Re: [OE-core] [PATCH v2] create-spdx: Don't collect natives sources

2021-09-24 Thread Joshua Watt 
On Fri, Sep 24, 2021, 2:16 AM Richard Purdie <
richard.pur...@linuxfoundation.org> wrote:

> On Thu, 2021-09-23 at 17:14 -0500, Joshua Watt wrote:
> > On 9/23/21 5:07 PM, Joshua Watt wrote:
> > >
> > > On 9/23/21 4:29 PM, Saul Wold wrote:
> > > >
> > > >
> > > > On 9/23/21 2:05 PM, Joshua Watt wrote:
> > > > >
> > > > > On 9/23/21 3:53 PM, Saul Wold wrote:
> > > > > > Extend the SPDXPackage to include is_native so it can be used
> later in
> > > > > > the processing.
> > > > > >
> > > > > > When the collect_dep_sources() runs, it collects sources from
> both
> > > > > > native
> > > > > > and non-native recipes. Later when the GENERATED_FROM matching
> > > > > > occurs it
> > > > > > may find the file (via checksum) from the native recipe since
> it's the
> > > > > > same checksum as the target file. The that are generated
> DocumentRefs
> > > > > > point to the native recipe rather than the target recipe
> DocumentRef.
> > > > > >
> > > > > > Signed-off-by: Saul Wold 
> > > > > > ---
> > > > > >   meta/classes/create-spdx.bbclass | 11 +--
> > > > > >   meta/lib/oe/spdx.py  |  1 +
> > > > > >   2 files changed, 10 insertions(+), 2 deletions(-)
> > > > > >
> > > > > > diff --git a/meta/classes/create-spdx.bbclass
> > > > > > b/meta/classes/create-spdx.bbclass
> > > > > > index 3c73c21c04..e565f0bf6c 100644
> > > > > > --- a/meta/classes/create-spdx.bbclass
> > > > > > +++ b/meta/classes/create-spdx.bbclass
> > > > > > @@ -336,6 +336,10 @@ def collect_dep_sources(d, dep_recipes):
> > > > > >   sources = {}
> > > > > >   for dep in dep_recipes:
> > > > > > +# Don't collect sources from native recipes as they
> > > > > > +# match non-native sources also.
> > > > > > +if dep.recipe.is_native == "True":
> > > > > > +continue
> > > > > >   recipe_files = set(dep.recipe.hasFiles)
> > > > > >   for spdx_file in dep.doc.files:
> > > > > > @@ -382,7 +386,6 @@ python do_create_spdx() {
> > > > > >   include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
> > > > > >   archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1"
> > > > > >   archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1"
> > > > > > -is_native = bb.data.inherits_class("native", d)
> > > > > >   creation_time =
> > > > > > datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
> > > > > > @@ -401,6 +404,10 @@ python do_create_spdx() {
> > > > > >   recipe.name = d.getVar("PN")
> > > > > >   recipe.versionInfo = d.getVar("PV")
> > > > > >   recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
> > > > > > +if bb.data.inherits_class("native", d):
> > > > > > +recipe.is_native = "True"
> > > > > > +else:
> > > > > > +recipe.is_native = "False"
> > > > > >   for s in d.getVar('SRC_URI').split():
> > > > > >   if not s.startswith("file://"):
> > > > > > @@ -480,7 +487,7 @@ python do_create_spdx() {
> > > > > >   sources = collect_dep_sources(d, dep_recipes)
> > > > > >   found_licenses = {license.name:recipe_ref.externalDocumentId
>
> > > > > > + ":" + license.licenseId for license in
> > > > > > doc.hasExtractedLicensingInfos}
> > > > > > -if not is_native:
> > > > > > +if recipe.is_native is "False":
> > > > > >   bb.build.exec_func("read_subpackage_metadata", d)
> > > > > >   pkgdest = Path(d.getVar("PKGDEST"))
> > > > > > diff --git a/meta/lib/oe/spdx.py b/meta/lib/oe/spdx.py
> > > > > > index 9814fbfd66..452148f339 100644
> > > > > > --- a/meta/lib/oe/spdx.py
> > > > > > +++ b/meta/lib/oe/spdx.py
> > > > > > @@ -164,6 +164,7 @@ class SPDXPackage(SPDXObject):
> > > > > >   packageVerificationCode =
> _Object(SPDXPackageVerificationCode)
> > > > > >   hasFiles = _StringList()
> > > > > >   packageFileName = _String()
> > > > > > +is_native = _String()
> > > > >
> > > > > It's probably not well documented in this file, but this has to
> > > > > match to the SPDX standard; we can't add arbitrary fields here.
> When
> > > > > I was referring to an "annotation" I was specifcially referring to
> > > > > an SPDX annotation:
> > > > >
> > > > I should have figured that!
> > > >
> > > > > https://spdx.github.io/spdx-spec/8-annotations/
> > > > >
> > > > > We'd need to decide on some schema for encoding the data in the
> > > > > annotation
> > > > >
> > > > So we need to create an SPDXAnnotation type on spdx.py and define
> > > > what we want in the AnnotationComment field?
> > >
> > > Exactly. We can use the tool field to indicate that this is data we
> > > care about for a specific annotation, then put JSON or something in
> > > the annotation itself.
> >
> >
> > Also, I forgot to mention but I found it really hard to convert the
> > normal SPDX spec document into the structure we need for JSON encoding
> > in spdx.py. I found it *much* easier to decipher the actual SPDX JSON
> > schema document:
> >
> >
>

Re: [OE-core] [PATCH v2] create-spdx: Don't collect natives sources

2021-09-24 Thread Richard Purdie 
On Thu, 2021-09-23 at 17:14 -0500, Joshua Watt wrote:
> On 9/23/21 5:07 PM, Joshua Watt wrote:
> > 
> > On 9/23/21 4:29 PM, Saul Wold wrote:
> > > 
> > > 
> > > On 9/23/21 2:05 PM, Joshua Watt wrote:
> > > > 
> > > > On 9/23/21 3:53 PM, Saul Wold wrote:
> > > > > Extend the SPDXPackage to include is_native so it can be used later in
> > > > > the processing.
> > > > > 
> > > > > When the collect_dep_sources() runs, it collects sources from both 
> > > > > native
> > > > > and non-native recipes. Later when the GENERATED_FROM matching 
> > > > > occurs it
> > > > > may find the file (via checksum) from the native recipe since it's the
> > > > > same checksum as the target file. The that are generated DocumentRefs
> > > > > point to the native recipe rather than the target recipe DocumentRef.
> > > > > 
> > > > > Signed-off-by: Saul Wold 
> > > > > ---
> > > > >   meta/classes/create-spdx.bbclass | 11 +--
> > > > >   meta/lib/oe/spdx.py  |  1 +
> > > > >   2 files changed, 10 insertions(+), 2 deletions(-)
> > > > > 
> > > > > diff --git a/meta/classes/create-spdx.bbclass 
> > > > > b/meta/classes/create-spdx.bbclass
> > > > > index 3c73c21c04..e565f0bf6c 100644
> > > > > --- a/meta/classes/create-spdx.bbclass
> > > > > +++ b/meta/classes/create-spdx.bbclass
> > > > > @@ -336,6 +336,10 @@ def collect_dep_sources(d, dep_recipes):
> > > > >   sources = {}
> > > > >   for dep in dep_recipes:
> > > > > +    # Don't collect sources from native recipes as they
> > > > > +    # match non-native sources also.
> > > > > +    if dep.recipe.is_native == "True":
> > > > > +    continue
> > > > >   recipe_files = set(dep.recipe.hasFiles)
> > > > >   for spdx_file in dep.doc.files:
> > > > > @@ -382,7 +386,6 @@ python do_create_spdx() {
> > > > >   include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
> > > > >   archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1"
> > > > >   archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1"
> > > > > -    is_native = bb.data.inherits_class("native", d)
> > > > >   creation_time = 
> > > > > datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
> > > > > @@ -401,6 +404,10 @@ python do_create_spdx() {
> > > > >   recipe.name = d.getVar("PN")
> > > > >   recipe.versionInfo = d.getVar("PV")
> > > > >   recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
> > > > > +    if bb.data.inherits_class("native", d):
> > > > > +    recipe.is_native = "True"
> > > > > +    else:
> > > > > +    recipe.is_native = "False"
> > > > >   for s in d.getVar('SRC_URI').split():
> > > > >   if not s.startswith("file://"):
> > > > > @@ -480,7 +487,7 @@ python do_create_spdx() {
> > > > >   sources = collect_dep_sources(d, dep_recipes)
> > > > >   found_licenses = {license.name:recipe_ref.externalDocumentId 
> > > > > + ":" + license.licenseId for license in 
> > > > > doc.hasExtractedLicensingInfos}
> > > > > -    if not is_native:
> > > > > +    if recipe.is_native is "False":
> > > > >   bb.build.exec_func("read_subpackage_metadata", d)
> > > > >   pkgdest = Path(d.getVar("PKGDEST"))
> > > > > diff --git a/meta/lib/oe/spdx.py b/meta/lib/oe/spdx.py
> > > > > index 9814fbfd66..452148f339 100644
> > > > > --- a/meta/lib/oe/spdx.py
> > > > > +++ b/meta/lib/oe/spdx.py
> > > > > @@ -164,6 +164,7 @@ class SPDXPackage(SPDXObject):
> > > > >   packageVerificationCode = _Object(SPDXPackageVerificationCode)
> > > > >   hasFiles = _StringList()
> > > > >   packageFileName = _String()
> > > > > +    is_native = _String()
> > > > 
> > > > It's probably not well documented in this file, but this has to 
> > > > match to the SPDX standard; we can't add arbitrary fields here. When 
> > > > I was referring to an "annotation" I was specifcially referring to 
> > > > an SPDX annotation:
> > > > 
> > > I should have figured that!
> > > 
> > > > https://spdx.github.io/spdx-spec/8-annotations/
> > > > 
> > > > We'd need to decide on some schema for encoding the data in the 
> > > > annotation
> > > > 
> > > So we need to create an SPDXAnnotation type on spdx.py and define 
> > > what we want in the AnnotationComment field?
> > 
> > Exactly. We can use the tool field to indicate that this is data we 
> > care about for a specific annotation, then put JSON or something in 
> > the annotation itself.
> 
> 
> Also, I forgot to mention but I found it really hard to convert the 
> normal SPDX spec document into the structure we need for JSON encoding 
> in spdx.py. I found it *much* easier to decipher the actual SPDX JSON 
> schema document:
> 
> https://github.com/spdx/spdx-spec/blob/development/v2.2.1/schemas/spdx-schema.json
> 
> 

Could someone put a few comments into the code just so that we don't forget some
of these constraints in future please?

Cheers,

Richard


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156289):

Re: [OE-core] [PATCH v2] create-spdx: Don't collect natives sources

2021-09-23 Thread Joshua Watt 


On 9/23/21 5:07 PM, Joshua Watt wrote:


On 9/23/21 4:29 PM, Saul Wold wrote:



On 9/23/21 2:05 PM, Joshua Watt wrote:


On 9/23/21 3:53 PM, Saul Wold wrote:

Extend the SPDXPackage to include is_native so it can be used later in
the processing.

When the collect_dep_sources() runs, it collects sources from both 
native
and non-native recipes. Later when the GENERATED_FROM matching 
occurs it

may find the file (via checksum) from the native recipe since it's the
same checksum as the target file. The that are generated DocumentRefs
point to the native recipe rather than the target recipe DocumentRef.

Signed-off-by: Saul Wold 
---
  meta/classes/create-spdx.bbclass | 11 +--
  meta/lib/oe/spdx.py  |  1 +
  2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass 
b/meta/classes/create-spdx.bbclass

index 3c73c21c04..e565f0bf6c 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -336,6 +336,10 @@ def collect_dep_sources(d, dep_recipes):
  sources = {}
  for dep in dep_recipes:
+    # Don't collect sources from native recipes as they
+    # match non-native sources also.
+    if dep.recipe.is_native == "True":
+    continue
  recipe_files = set(dep.recipe.hasFiles)
  for spdx_file in dep.doc.files:
@@ -382,7 +386,6 @@ python do_create_spdx() {
  include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
  archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1"
  archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1"
-    is_native = bb.data.inherits_class("native", d)
  creation_time = 
datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")

@@ -401,6 +404,10 @@ python do_create_spdx() {
  recipe.name = d.getVar("PN")
  recipe.versionInfo = d.getVar("PV")
  recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
+    if bb.data.inherits_class("native", d):
+    recipe.is_native = "True"
+    else:
+    recipe.is_native = "False"
  for s in d.getVar('SRC_URI').split():
  if not s.startswith("file://"):
@@ -480,7 +487,7 @@ python do_create_spdx() {
  sources = collect_dep_sources(d, dep_recipes)
  found_licenses = {license.name:recipe_ref.externalDocumentId 
+ ":" + license.licenseId for license in 
doc.hasExtractedLicensingInfos}

-    if not is_native:
+    if recipe.is_native is "False":
  bb.build.exec_func("read_subpackage_metadata", d)
  pkgdest = Path(d.getVar("PKGDEST"))
diff --git a/meta/lib/oe/spdx.py b/meta/lib/oe/spdx.py
index 9814fbfd66..452148f339 100644
--- a/meta/lib/oe/spdx.py
+++ b/meta/lib/oe/spdx.py
@@ -164,6 +164,7 @@ class SPDXPackage(SPDXObject):
  packageVerificationCode = _Object(SPDXPackageVerificationCode)
  hasFiles = _StringList()
  packageFileName = _String()
+    is_native = _String()


It's probably not well documented in this file, but this has to 
match to the SPDX standard; we can't add arbitrary fields here. When 
I was referring to an "annotation" I was specifcially referring to 
an SPDX annotation:



I should have figured that!


https://spdx.github.io/spdx-spec/8-annotations/

We'd need to decide on some schema for encoding the data in the 
annotation


So we need to create an SPDXAnnotation type on spdx.py and define 
what we want in the AnnotationComment field?


Exactly. We can use the tool field to indicate that this is data we 
care about for a specific annotation, then put JSON or something in 
the annotation itself.



Also, I forgot to mention but I found it really hard to convert the 
normal SPDX spec document into the structure we need for JSON encoding 
in spdx.py. I found it *much* easier to decipher the actual SPDX JSON 
schema document:


https://github.com/spdx/spdx-spec/blob/development/v2.2.1/schemas/spdx-schema.json





Sua!


  class SPDXFile(SPDXObject):



-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156281): 
https://lists.openembedded.org/g/openembedded-core/message/156281
Mute This Topic: https://lists.openembedded.org/mt/85824376/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH v2] create-spdx: Don't collect natives sources

2021-09-23 Thread Joshua Watt 


On 9/23/21 4:29 PM, Saul Wold wrote:



On 9/23/21 2:05 PM, Joshua Watt wrote:


On 9/23/21 3:53 PM, Saul Wold wrote:

Extend the SPDXPackage to include is_native so it can be used later in
the processing.

When the collect_dep_sources() runs, it collects sources from both 
native
and non-native recipes. Later when the GENERATED_FROM matching 
occurs it

may find the file (via checksum) from the native recipe since it's the
same checksum as the target file. The that are generated DocumentRefs
point to the native recipe rather than the target recipe DocumentRef.

Signed-off-by: Saul Wold 
---
  meta/classes/create-spdx.bbclass | 11 +--
  meta/lib/oe/spdx.py  |  1 +
  2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass 
b/meta/classes/create-spdx.bbclass

index 3c73c21c04..e565f0bf6c 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -336,6 +336,10 @@ def collect_dep_sources(d, dep_recipes):
  sources = {}
  for dep in dep_recipes:
+    # Don't collect sources from native recipes as they
+    # match non-native sources also.
+    if dep.recipe.is_native == "True":
+    continue
  recipe_files = set(dep.recipe.hasFiles)
  for spdx_file in dep.doc.files:
@@ -382,7 +386,6 @@ python do_create_spdx() {
  include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
  archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1"
  archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1"
-    is_native = bb.data.inherits_class("native", d)
  creation_time = 
datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")

@@ -401,6 +404,10 @@ python do_create_spdx() {
  recipe.name = d.getVar("PN")
  recipe.versionInfo = d.getVar("PV")
  recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
+    if bb.data.inherits_class("native", d):
+    recipe.is_native = "True"
+    else:
+    recipe.is_native = "False"
  for s in d.getVar('SRC_URI').split():
  if not s.startswith("file://"):
@@ -480,7 +487,7 @@ python do_create_spdx() {
  sources = collect_dep_sources(d, dep_recipes)
  found_licenses = {license.name:recipe_ref.externalDocumentId + 
":" + license.licenseId for license in doc.hasExtractedLicensingInfos}

-    if not is_native:
+    if recipe.is_native is "False":
  bb.build.exec_func("read_subpackage_metadata", d)
  pkgdest = Path(d.getVar("PKGDEST"))
diff --git a/meta/lib/oe/spdx.py b/meta/lib/oe/spdx.py
index 9814fbfd66..452148f339 100644
--- a/meta/lib/oe/spdx.py
+++ b/meta/lib/oe/spdx.py
@@ -164,6 +164,7 @@ class SPDXPackage(SPDXObject):
  packageVerificationCode = _Object(SPDXPackageVerificationCode)
  hasFiles = _StringList()
  packageFileName = _String()
+    is_native = _String()


It's probably not well documented in this file, but this has to match 
to the SPDX standard; we can't add arbitrary fields here. When I was 
referring to an "annotation" I was specifcially referring to an SPDX 
annotation:



I should have figured that!


https://spdx.github.io/spdx-spec/8-annotations/

We'd need to decide on some schema for encoding the data in the 
annotation


So we need to create an SPDXAnnotation type on spdx.py and define what 
we want in the AnnotationComment field?


Exactly. We can use the tool field to indicate that this is data we care 
about for a specific annotation, then put JSON or something in the 
annotation itself.




Sua!


  class SPDXFile(SPDXObject):



-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156280): 
https://lists.openembedded.org/g/openembedded-core/message/156280
Mute This Topic: https://lists.openembedded.org/mt/85824376/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH v2] create-spdx: Don't collect natives sources

2021-09-23 Thread Saul Wold 



On 9/23/21 2:05 PM, Joshua Watt wrote:


On 9/23/21 3:53 PM, Saul Wold wrote:

Extend the SPDXPackage to include is_native so it can be used later in
the processing.

When the collect_dep_sources() runs, it collects sources from both native
and non-native recipes. Later when the GENERATED_FROM matching occurs it
may find the file (via checksum) from the native recipe since it's the
same checksum as the target file. The that are generated DocumentRefs
point to the native recipe rather than the target recipe DocumentRef.

Signed-off-by: Saul Wold 
---
  meta/classes/create-spdx.bbclass | 11 +--
  meta/lib/oe/spdx.py  |  1 +
  2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass 
b/meta/classes/create-spdx.bbclass

index 3c73c21c04..e565f0bf6c 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -336,6 +336,10 @@ def collect_dep_sources(d, dep_recipes):
  sources = {}
  for dep in dep_recipes:
+    # Don't collect sources from native recipes as they
+    # match non-native sources also.
+    if dep.recipe.is_native == "True":
+    continue
  recipe_files = set(dep.recipe.hasFiles)
  for spdx_file in dep.doc.files:
@@ -382,7 +386,6 @@ python do_create_spdx() {
  include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
  archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1"
  archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1"
-    is_native = bb.data.inherits_class("native", d)
  creation_time = 
datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")

@@ -401,6 +404,10 @@ python do_create_spdx() {
  recipe.name = d.getVar("PN")
  recipe.versionInfo = d.getVar("PV")
  recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
+    if bb.data.inherits_class("native", d):
+    recipe.is_native = "True"
+    else:
+    recipe.is_native = "False"
  for s in d.getVar('SRC_URI').split():
  if not s.startswith("file://"):
@@ -480,7 +487,7 @@ python do_create_spdx() {
  sources = collect_dep_sources(d, dep_recipes)
  found_licenses = {license.name:recipe_ref.externalDocumentId + 
":" + license.licenseId for license in doc.hasExtractedLicensingInfos}

-    if not is_native:
+    if recipe.is_native is "False":
  bb.build.exec_func("read_subpackage_metadata", d)
  pkgdest = Path(d.getVar("PKGDEST"))
diff --git a/meta/lib/oe/spdx.py b/meta/lib/oe/spdx.py
index 9814fbfd66..452148f339 100644
--- a/meta/lib/oe/spdx.py
+++ b/meta/lib/oe/spdx.py
@@ -164,6 +164,7 @@ class SPDXPackage(SPDXObject):
  packageVerificationCode = _Object(SPDXPackageVerificationCode)
  hasFiles = _StringList()
  packageFileName = _String()
+    is_native = _String()


It's probably not well documented in this file, but this has to match to 
the SPDX standard; we can't add arbitrary fields here. When I was 
referring to an "annotation" I was specifcially referring to an SPDX 
annotation:



I should have figured that!


https://spdx.github.io/spdx-spec/8-annotations/

We'd need to decide on some schema for encoding the data in the annotation

So we need to create an SPDXAnnotation type on spdx.py and define what 
we want in the AnnotationComment field?


Sua!


  class SPDXFile(SPDXObject):


--
Sau!

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156277): 
https://lists.openembedded.org/g/openembedded-core/message/156277
Mute This Topic: https://lists.openembedded.org/mt/85824376/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH v2] create-spdx: Don't collect natives sources

2021-09-23 Thread Joshua Watt 


On 9/23/21 3:53 PM, Saul Wold wrote:

Extend the SPDXPackage to include is_native so it can be used later in
the processing.

When the collect_dep_sources() runs, it collects sources from both native
and non-native recipes. Later when the GENERATED_FROM matching occurs it
may find the file (via checksum) from the native recipe since it's the
same checksum as the target file. The that are generated DocumentRefs
point to the native recipe rather than the target recipe DocumentRef.

Signed-off-by: Saul Wold 
---
  meta/classes/create-spdx.bbclass | 11 +--
  meta/lib/oe/spdx.py  |  1 +
  2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 3c73c21c04..e565f0bf6c 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -336,6 +336,10 @@ def collect_dep_sources(d, dep_recipes):
  
  sources = {}

  for dep in dep_recipes:
+# Don't collect sources from native recipes as they
+# match non-native sources also.
+if dep.recipe.is_native == "True":
+continue
  recipe_files = set(dep.recipe.hasFiles)
  
  for spdx_file in dep.doc.files:

@@ -382,7 +386,6 @@ python do_create_spdx() {
  include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
  archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1"
  archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1"
-is_native = bb.data.inherits_class("native", d)
  
  creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
  
@@ -401,6 +404,10 @@ python do_create_spdx() {

  recipe.name = d.getVar("PN")
  recipe.versionInfo = d.getVar("PV")
  recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
+if bb.data.inherits_class("native", d):
+recipe.is_native = "True"
+else:
+recipe.is_native = "False"
  
  for s in d.getVar('SRC_URI').split():

  if not s.startswith("file://"):
@@ -480,7 +487,7 @@ python do_create_spdx() {
  sources = collect_dep_sources(d, dep_recipes)
  found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + 
license.licenseId for license in doc.hasExtractedLicensingInfos}
  
-if not is_native:

+if recipe.is_native is "False":
  bb.build.exec_func("read_subpackage_metadata", d)
  
  pkgdest = Path(d.getVar("PKGDEST"))

diff --git a/meta/lib/oe/spdx.py b/meta/lib/oe/spdx.py
index 9814fbfd66..452148f339 100644
--- a/meta/lib/oe/spdx.py
+++ b/meta/lib/oe/spdx.py
@@ -164,6 +164,7 @@ class SPDXPackage(SPDXObject):
  packageVerificationCode = _Object(SPDXPackageVerificationCode)
  hasFiles = _StringList()
  packageFileName = _String()
+is_native = _String()


It's probably not well documented in this file, but this has to match to 
the SPDX standard; we can't add arbitrary fields here. When I was 
referring to an "annotation" I was specifcially referring to an SPDX 
annotation:


https://spdx.github.io/spdx-spec/8-annotations/

We'd need to decide on some schema for encoding the data in the annotation

  
  
  class SPDXFile(SPDXObject):

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156276): 
https://lists.openembedded.org/g/openembedded-core/message/156276
Mute This Topic: https://lists.openembedded.org/mt/85824376/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH v2] create-spdx: Don't collect natives sources

2021-09-23 Thread Saul Wold 
Extend the SPDXPackage to include is_native so it can be used later in
the processing.

When the collect_dep_sources() runs, it collects sources from both native
and non-native recipes. Later when the GENERATED_FROM matching occurs it
may find the file (via checksum) from the native recipe since it's the
same checksum as the target file. The that are generated DocumentRefs
point to the native recipe rather than the target recipe DocumentRef.

Signed-off-by: Saul Wold 
---
 meta/classes/create-spdx.bbclass | 11 +--
 meta/lib/oe/spdx.py  |  1 +
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 3c73c21c04..e565f0bf6c 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -336,6 +336,10 @@ def collect_dep_sources(d, dep_recipes):
 
 sources = {}
 for dep in dep_recipes:
+# Don't collect sources from native recipes as they
+# match non-native sources also.
+if dep.recipe.is_native == "True":
+continue
 recipe_files = set(dep.recipe.hasFiles)
 
 for spdx_file in dep.doc.files:
@@ -382,7 +386,6 @@ python do_create_spdx() {
 include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
 archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1"
 archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1"
-is_native = bb.data.inherits_class("native", d)
 
 creation_time = 
datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
@@ -401,6 +404,10 @@ python do_create_spdx() {
 recipe.name = d.getVar("PN")
 recipe.versionInfo = d.getVar("PV")
 recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
+if bb.data.inherits_class("native", d):
+recipe.is_native = "True"
+else:
+recipe.is_native = "False"
 
 for s in d.getVar('SRC_URI').split():
 if not s.startswith("file://"):
@@ -480,7 +487,7 @@ python do_create_spdx() {
 sources = collect_dep_sources(d, dep_recipes)
 found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + 
license.licenseId for license in doc.hasExtractedLicensingInfos}
 
-if not is_native:
+if recipe.is_native is "False":
 bb.build.exec_func("read_subpackage_metadata", d)
 
 pkgdest = Path(d.getVar("PKGDEST"))
diff --git a/meta/lib/oe/spdx.py b/meta/lib/oe/spdx.py
index 9814fbfd66..452148f339 100644
--- a/meta/lib/oe/spdx.py
+++ b/meta/lib/oe/spdx.py
@@ -164,6 +164,7 @@ class SPDXPackage(SPDXObject):
 packageVerificationCode = _Object(SPDXPackageVerificationCode)
 hasFiles = _StringList()
 packageFileName = _String()
+is_native = _String()
 
 
 class SPDXFile(SPDXObject):
-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156275): 
https://lists.openembedded.org/g/openembedded-core/message/156275
Mute This Topic: https://lists.openembedded.org/mt/85824376/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-