[OE-core] [PATCH] u-boot: Enable devupstream

2024-05-02 Thread Peter Hoyes 
To enable downstream testing of the bleeding edge of U-Boot,
add devupstream to the recipe, with SRCREV set to automatically
track the latest revision.

Signed-off-by: Peter Hoyes 
---
 meta/recipes-bsp/u-boot/u-boot_2024.04.bb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/meta/recipes-bsp/u-boot/u-boot_2024.04.bb 
b/meta/recipes-bsp/u-boot/u-boot_2024.04.bb
index b15bcaa818..dc2e2da8ae 100644
--- a/meta/recipes-bsp/u-boot/u-boot_2024.04.bb
+++ b/meta/recipes-bsp/u-boot/u-boot_2024.04.bb
@@ -3,3 +3,6 @@ require u-boot.inc
 
 DEPENDS += "bc-native dtc-native python3-pyelftools-native"
 
+BBCLASSEXTEND += "devupstream:target"
+SRCREV:class-devupstream = "${AUTOREV}"
+PV:class-devupstream = "dev+git"
-- 
2.34.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#198911): 
https://lists.openembedded.org/g/openembedded-core/message/198911
Mute This Topic: https://lists.openembedded.org/mt/105860305/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH] u-boot-tools: Package mkeficapsule

2024-04-09 Thread Peter Hoyes 
mkeficapsule is a tool provided by U-Boot (as part of the tools-only
targets) for generating UEFI capsule update archives.

Install mkeficapsule into a u-boot-tools-mkeficapsule package.

Signed-off-by: Peter Hoyes 
---
 meta/recipes-bsp/u-boot/u-boot-tools.inc | 13 +
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/meta/recipes-bsp/u-boot/u-boot-tools.inc 
b/meta/recipes-bsp/u-boot/u-boot-tools.inc
index 09b3c3f68c..f3010763c0 100644
--- a/meta/recipes-bsp/u-boot/u-boot-tools.inc
+++ b/meta/recipes-bsp/u-boot/u-boot-tools.inc
@@ -4,10 +4,10 @@ DEPENDS += "gnutls openssl util-linux swig-native"
 inherit python3native
 export STAGING_INCDIR="${STAGING_INCDIR_NATIVE}"
 
-PROVIDES = "${MLPREFIX}u-boot-mkimage ${MLPREFIX}u-boot-mkenvimage"
-PROVIDES:class-native = "u-boot-mkimage-native u-boot-mkenvimage-native"
+PROVIDES = "${MLPREFIX}u-boot-mkimage ${MLPREFIX}u-boot-mkenvimage 
${MLPREFIX}u-boot-mkeficapsule"
+PROVIDES:class-native = "u-boot-mkimage-native u-boot-mkenvimage-native 
u-boot-mkeficapsule-native"
 
-PACKAGES += "${PN}-mkimage ${PN}-mkenvimage"
+PACKAGES += "${PN}-mkimage ${PN}-mkenvimage ${PN}-mkeficapsule"
 
 # Required for backward compatibility with "u-boot-mkimage-xxx.bb"
 RPROVIDES:${PN}-mkimage = "u-boot-mkimage"
@@ -70,15 +70,20 @@ do_install () {
# fit_check_sign
install -m 0755 tools/fit_check_sign ${D}${bindir}/uboot-fit_check_sign
ln -sf uboot-fit_check_sign ${D}${bindir}/fit_check_sign
+
+   # mkeficapsule
+   install -m 0755 tools/mkeficapsule ${D}${bindir}/uboot-mkeficapsule
+   ln -sf uboot-mkeficapsule ${D}${bindir}/mkeficapsule
 }
 
 ALLOW_EMPTY:${PN} = "1"
 FILES:${PN} = ""
 FILES:${PN}-mkimage = "${bindir}/uboot-mkimage ${bindir}/mkimage 
${bindir}/uboot-dumpimage ${bindir}/dumpimage ${bindir}/uboot-fit_check_sign 
${bindir}/fit_check_sign"
 FILES:${PN}-mkenvimage = "${bindir}/uboot-mkenvimage ${bindir}/mkenvimage"
+FILES:${PN}-mkeficapsule = "${bindir}/uboot-mkeficapsule 
${bindir}/mkeficapsule"
 
 RDEPENDS:${PN}-mkimage += "dtc"
-RDEPENDS:${PN} += "${PN}-mkimage ${PN}-mkenvimage"
+RDEPENDS:${PN} += "${PN}-mkimage ${PN}-mkenvimage ${PN}-mkeficapsule"
 RDEPENDS:${PN}:class-native = ""
 
 BBCLASSEXTEND = "native nativesdk"
-- 
2.34.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#198055): 
https://lists.openembedded.org/g/openembedded-core/message/198055
Mute This Topic: https://lists.openembedded.org/mt/105417779/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH v3] rootfs-postcommands: Set vardeps for do_write_test_data

2023-06-29 Thread Peter Hoyes 
From: Peter Hoyes 

The testdata.json file generated by testdata.bbclass currently contains
almost all Bitbake variables. However only a small number of variables
are actually used and the testdata.json is not automatically updated
when the variables are updated.

Introduce the testdata_vars bbclass which defines TESTDATA_EXPORT_VARS
to explicitly set the variables to be collected for testdata.json and
populate with all variables used by runtime and sdk tests in OE-core.
Use this variable to set the vardeps of do_write_test_data.

Modify export2json to take an explicit list of keys to export, instead
of exporting everything in the datastore.

Adapt write_sdk_test_data and meta-ide-support to use the same list of
testdata variables.

Add a demonstrative OE selftest to rootfspostcommandstests.

Signed-off-by: Peter Hoyes 
---

Changes from v2:
 * Move TESTDATA_EXPORT_VARS to common testdata_vars.bbclass
 * Update other export2json call sites (eSDK and meta-ide-support)

Tested with:
 * oe-selftest -r rootfspostcommandstests
 * oe-selftest -r sstatetests.SStateHashSameSigs
 * bitbake core-image-sato -c testimage
 * bitbake core-image-sato -c testsdkext
 * bitbake buildtools-tarball buildtools-extended-tarball
 * bitbake meta-ide-support

 meta/classes-recipe/populate_sdk_base.bbclass |  5 +++-
 .../rootfs-postcommands.bbclass   |  6 +++-
 meta/classes-recipe/testdata_vars.bbclass | 30 +++
 meta/lib/oe/data.py   | 17 +--
 .../selftest/cases/rootfspostcommandstests.py | 19 
 meta/recipes-core/meta/meta-ide-support.bb|  5 +++-
 6 files changed, 63 insertions(+), 19 deletions(-)
 create mode 100644 meta/classes-recipe/testdata_vars.bbclass

diff --git a/meta/classes-recipe/populate_sdk_base.bbclass 
b/meta/classes-recipe/populate_sdk_base.bbclass
index b23ea26d40..ffb06f2062 100644
--- a/meta/classes-recipe/populate_sdk_base.bbclass
+++ b/meta/classes-recipe/populate_sdk_base.bbclass
@@ -134,11 +134,14 @@ sdk_prune_dirs () {
 done
 }
 
+inherit testdata_vars
+
 python write_sdk_test_data() {
 from oe.data import export2json
 testdata = "%s/%s.testdata.json" % (d.getVar('SDKDEPLOYDIR'), 
d.getVar('TOOLCHAIN_OUTPUTNAME'))
 bb.utils.mkdirhier(os.path.dirname(testdata))
-export2json(d, testdata)
+keys = d.getVar('TESTDATA_EXPORT_VARS').split()
+export2json(d, keys, testdata)
 }
 
 python write_host_sdk_manifest () {
diff --git a/meta/classes-recipe/rootfs-postcommands.bbclass 
b/meta/classes-recipe/rootfs-postcommands.bbclass
index 652601b95f..26eddc8c35 100644
--- a/meta/classes-recipe/rootfs-postcommands.bbclass
+++ b/meta/classes-recipe/rootfs-postcommands.bbclass
@@ -454,6 +454,8 @@ rootfs_sysroot_relativelinks () {
 }
 
 # Generated test data json file
+inherit testdata_vars
+
 python write_image_test_data() {
 from oe.data import export2json
 
@@ -461,8 +463,9 @@ python write_image_test_data() {
 link_name = d.getVar('IMAGE_LINK_NAME')
 testdata_name = os.path.join(deploy_dir, "%s.testdata.json" % 
d.getVar('IMAGE_NAME'))
 
+keys = d.getVar('TESTDATA_EXPORT_VARS').split()
 searchString = "%s/"%(d.getVar("TOPDIR")).replace("//","/")
-export2json(d, testdata_name, searchString=searchString, replaceString="")
+export2json(d, keys, testdata_name, searchString=searchString, 
replaceString="")
 
 if os.path.exists(testdata_name) and link_name:
 testdata_link = os.path.join(deploy_dir, "%s.testdata.json" % 
link_name)
@@ -471,6 +474,7 @@ python write_image_test_data() {
 os.remove(testdata_link)
 os.symlink(os.path.basename(testdata_name), testdata_link)
 }
+write_image_test_data[vardeps] += "${TESTDATA_EXPORT_VARS}"
 write_image_test_data[vardepsexclude] += "TOPDIR"
 
 # Check for unsatisfied recommendations (RRECOMMENDS)
diff --git a/meta/classes-recipe/testdata_vars.bbclass 
b/meta/classes-recipe/testdata_vars.bbclass
new file mode 100644
index 00..cecbc6d7ca
--- /dev/null
+++ b/meta/classes-recipe/testdata_vars.bbclass
@@ -0,0 +1,30 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+# The variables exported to testdata.json for testimage and other similar tasks
+TESTDATA_EXPORT_VARS ?= "\
+ARCH \
+DEPLOY_DIR \
+DEPLOY_DIR_DEB \
+DISTRO_FEATURES \
+DL_DIR \
+IMAGE \
+IMAGE_FEATURES \
+libdir \
+MACHINE \
+MULTILIB_VARIANTS \
+PACKAGE_FEED_GPG_NAME \
+PTEST_EXPECT_FAILURE \
+QEMU_USE_KVM \
+SDK_DEPLOY \
+SDKPATH \
+T \
+TEST_LOG_DIR \
+TOOLCHAINEXT_OUTPUTNAME \
+TUNE_PKGARCH \
+VIRTUAL-RUNTIME_init_manager \
+WORKDIR \
+"
diff --git a/meta/lib/oe/data.py b/meta/lib/oe/data.py
index 37121cfad2..731f2b5b46 100644
--- a/meta/lib/oe/data.py
+++ b/meta/lib/oe/data.py
@@ -23,23 +23,8 @@ def typed_val

Re: [OE-core] [PATCH] rootfs-postcommands: Set vardeps for write_image_test_data

2023-06-22 Thread Peter Hoyes 


On 19/06/2023 13:07, Richard Purdie wrote:

On Mon, 2023-06-19 at 12:47 +0100, Peter Hoyes wrote:
  
It sounds like moving testdata creation to a separate task doesn't

bring much benefit so I'll revert to something based off this patch.

It looks like a relatively small number of variables are actually
used by OEQA tests in OE-core:
  
  * IMAGE

  * ARCH
  * MACHINE
  * PACKAGE_FEED_GPG_NAME
  * VIRTUAL-RUNTIME_init_manager
  * WORKDIR
  * DISTRO_FEATURES
  * PTEST_EXPECT_FAILURE
  * TEST_LOG_DIR
  * libdir
  * T
  * MULTILIB_VARIANTS
  * SDKPATH
  * SDK_DEPLOY
  * TOOLCHAINEXT_OUTPUTNAME
  * QEMU_USE_KVM
 
  * DL_DIR
 
Notably, DATE/TIME do not seem to be referenced at all (apart from a

comment in oeqa/runtime/cases/ptest.py recommending *not* using
DATETIME for test output). Additionally, DL_DIR and WORKDIR are
updated on each testimage execution using TESTIMAGE_UPDATE_VARS.
  
  
How about introducing a new variable, e.g. TESTIMAGE_EXPORT_VARS,

containing just the list above (to which other variables could
optionally be appended in other layers)?


I think that is certainly worth trying.

Cheers,

Richard


This is implemented in v2.

I have validated the new patch locally on:

 * The new test case
 * sstatetests.SStateHashSameSigs
 * qemuarm64 core-image-sato testimage
 * qemuarm64 core-image-minimal-dev testimage

Thanks,

Peter


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#183271): 
https://lists.openembedded.org/g/openembedded-core/message/183271
Mute This Topic: https://lists.openembedded.org/mt/98951993/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH v2] rootfs-postcommands: Set vardeps for write_image_test_data

2023-06-22 Thread Peter Hoyes 
From: Peter Hoyes 

The testdata.json file generated as part of the rootfs postprocess
commands currently contains almost all Bitbake variables and is used by
OEQA test cases to inspect the build environment. However only a small
number of variables are actually used and the testdata.json is not
automatically updated when the variables are updated.

Introduce the TESTIMAGE_EXPORT_VARS variable to explicitly define the
variables to be collected for testdata.json and populate with all
variables used by runtime tests in OE-core. Use this variable to set the
vardeps of write_image_test_data.

Modify export2json to take an explicit list of keys to export, instead
of exporting everything in the datastore.

Add a demonstrative OE selftest to rootfspostcommandstests.

Signed-off-by: Peter Hoyes 
---
 .../rootfs-postcommands.bbclass   | 28 ++-
 meta/lib/oe/data.py   | 19 ++---
 .../selftest/cases/rootfspostcommandstests.py | 19 +
 3 files changed, 48 insertions(+), 18 deletions(-)

diff --git a/meta/classes-recipe/rootfs-postcommands.bbclass 
b/meta/classes-recipe/rootfs-postcommands.bbclass
index 652601b95f..cb5b0727b8 100644
--- a/meta/classes-recipe/rootfs-postcommands.bbclass
+++ b/meta/classes-recipe/rootfs-postcommands.bbclass
@@ -453,6 +453,30 @@ rootfs_sysroot_relativelinks () {
sysroot-relativelinks.py ${SDK_OUTPUT}/${SDKTARGETSYSROOT}
 }
 
+TESTIMAGE_EXPORT_VARS ?= "\
+ARCH \
+DEPLOY_DIR \
+DEPLOY_DIR_DEB \
+DISTRO_FEATURES \
+DL_DIR \
+IMAGE \
+IMAGE_FEATURES \
+libdir \
+MACHINE \
+MULTILIB_VARIANTS \
+PACKAGE_FEED_GPG_NAME \
+PTEST_EXPECT_FAILURE \
+QEMU_USE_KVM \
+SDK_DEPLOY \
+SDKPATH \
+T \
+TEST_LOG_DIR \
+TOOLCHAINEXT_OUTPUTNAME \
+TUNE_PKGARCH \
+VIRTUAL-RUNTIME_init_manager \
+WORKDIR \
+"
+
 # Generated test data json file
 python write_image_test_data() {
 from oe.data import export2json
@@ -462,7 +486,8 @@ python write_image_test_data() {
 testdata_name = os.path.join(deploy_dir, "%s.testdata.json" % 
d.getVar('IMAGE_NAME'))
 
 searchString = "%s/"%(d.getVar("TOPDIR")).replace("//","/")
-export2json(d, testdata_name, searchString=searchString, replaceString="")
+exportkeys = d.getVar('TESTIMAGE_EXPORT_VARS').split()
+export2json(d, exportkeys, testdata_name, searchString=searchString, 
replaceString="")
 
 if os.path.exists(testdata_name) and link_name:
 testdata_link = os.path.join(deploy_dir, "%s.testdata.json" % 
link_name)
@@ -471,6 +496,7 @@ python write_image_test_data() {
 os.remove(testdata_link)
 os.symlink(os.path.basename(testdata_name), testdata_link)
 }
+write_image_test_data[vardeps] += "${TESTIMAGE_EXPORT_VARS}"
 write_image_test_data[vardepsexclude] += "TOPDIR"
 
 # Check for unsatisfied recommendations (RRECOMMENDS)
diff --git a/meta/lib/oe/data.py b/meta/lib/oe/data.py
index 37121cfad2..101a52e5e0 100644
--- a/meta/lib/oe/data.py
+++ b/meta/lib/oe/data.py
@@ -23,25 +23,10 @@ def typed_value(key, d):
 except (TypeError, ValueError) as exc:
 bb.msg.fatal("Data", "%s: %s" % (key, str(exc)))
 
-def export2json(d, json_file, expand=True, searchString="",replaceString=""):
+def export2json(d, keys, json_file, expand=True, 
searchString="",replaceString=""):
 data2export = {}
-keys2export = []
 
-for key in d.keys():
-if key.startswith("_"):
-continue
-elif key.startswith("BB"):
-continue
-elif key.startswith("B_pn"):
-continue
-elif key.startswith("do_"):
-continue
-elif d.getVarFlag(key, "func"):
-continue
-
-keys2export.append(key)
-
-for key in keys2export:
+for key in keys:
 try:
 data2export[key] = d.getVar(key, 
expand).replace(searchString,replaceString)
 except bb.data_smart.ExpansionError:
diff --git a/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py 
b/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py
index 44e2c09a6f..d15e766f49 100644
--- a/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py
+++ b/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py
@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
+import json
 import os
 import oe
 import unittest
@@ -95,3 +96,21 @@ class ShadowUtilsTidyFiles(OESelftestTestCase):
 unsorted.append(file)
 if (unsorted):
 raise Exception("The following files were not sorted by ID as 
expected: %s" % unsorted)
+
+
+class TestDataTests(OESelftestTestCase):
+def test_vardeps(self):
+"""
+Test that variables changes are

Re: [OE-core] [PATCH] rootfs-postcommands: Set vardeps for write_image_test_data

2023-06-19 Thread Peter Hoyes 

On 13/06/2023 12:33, Richard Purdie wrote:

On Tue, 2023-06-13 at 10:55 +0100, Peter Hoyes wrote:

On 23/05/2023 18:06, Ross Burton wrote:

On 17 May 2023, at 17:29, Peter Hoyes via lists.openembedded.org
  wrote:

The testdata.json file generated as part of the rootfs
postprocess
commands contains almost all Bitbake variables and is used by
OEQA test
cases to inspect the build environment. However it is only
regenerated
when the rootfs task is otherwise retriggered, complicating the
process
of developing OEQA test cases.

I’m about to run for dinner, but would it make more sense for this
function to be a separate task inside testimage.bbclass that
happens after rootfs?  This will stop the explosion of variable
dependencies causing rootfs to re-run.

Ross

Thanks for the suggestion - I have implemented this for consideration
in
https://lists.openembedded.org/g/openembedded-core/topic/patch_1_2/99501785

Making it a separate task is fine, the challenge is that it needs to
run at the same time as image processing so that DATE/TIME are
consistent and the image names are correct.

That implies that it has to be before do_image_complete which means any
change in variables will retrigger the image creation :(.

I think we need to try and fix this differently and stop exporting
every variable.

Cheers,

Richard


If I understand correctly...

It sounds like moving testdata creation to a separate task doesn't bring 
much benefit so I'll revert to something based off this patch.


It looks like a relatively small number of variables are actually used 
by OEQA tests in OE-core:


 * IMAGE
 * ARCH
 * MACHINE
 * PACKAGE_FEED_GPG_NAME
 * VIRTUAL-RUNTIME_init_manager
 * WORKDIR
 * DISTRO_FEATURES
 * PTEST_EXPECT_FAILURE
 * TEST_LOG_DIR
 * libdir
 * T
 * MULTILIB_VARIANTS
 * SDKPATH
 * SDK_DEPLOY
 * TOOLCHAINEXT_OUTPUTNAME
 * QEMU_USE_KVM
 * DL_DIR

Notably, DATE/TIME do not seem to be referenced at all (apart from a 
comment in oeqa/runtime/cases/ptest.py recommending *not* using DATETIME 
for test output). Additionally, DL_DIR and WORKDIR are updated on each 
testimage execution using TESTIMAGE_UPDATE_VARS.


How about introducing a new variable, e.g. TESTIMAGE_EXPORT_VARS, 
containing just the list above (to which other variables could 
optionally be appended in other layers)?


Peter

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#183101): 
https://lists.openembedded.org/g/openembedded-core/message/183101
Mute This Topic: https://lists.openembedded.org/mt/98951993/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] rootfs-postcommands: Set vardeps for write_image_test_data

2023-06-13 Thread Peter Hoyes 


On 23/05/2023 18:06, Ross Burton wrote:

On 17 May 2023, at 17:29, Peter Hoyes via lists.openembedded.org 
 wrote:

The testdata.json file generated as part of the rootfs postprocess
commands contains almost all Bitbake variables and is used by OEQA test
cases to inspect the build environment. However it is only regenerated
when the rootfs task is otherwise retriggered, complicating the process
of developing OEQA test cases.

I’m about to run for dinner, but would it make more sense for this function to 
be a separate task inside testimage.bbclass that happens after rootfs?  This 
will stop the explosion of variable dependencies causing rootfs to re-run.

Ross


Thanks for the suggestion - I have implemented this for consideration in 
https://lists.openembedded.org/g/openembedded-core/topic/patch_1_2/99501785


Peter


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#182706): 
https://lists.openembedded.org/g/openembedded-core/message/182706
Mute This Topic: https://lists.openembedded.org/mt/98951993/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH 2/2] testimage: Set nostamp for do_write_image_test_data

2023-06-13 Thread Peter Hoyes 
From: Peter Hoyes 

The testdata.json file generated as part of the rootfs postprocess
commands contains almost all Bitbake variables and is used by OEQA test
cases to inspect the build environment. However it is not automatically
regenerated, complicating the process of developing OEQA test cases.

Set nostamp on the new do_write_image_test_data task so that it always
runs.

Add a demonstrative OE selftest.

Signed-off-by: Peter Hoyes 
---
 meta/classes-recipe/testimage.bbclass |  1 +
 meta/lib/oeqa/selftest/cases/testimage.py | 26 +++
 2 files changed, 27 insertions(+)
 create mode 100644 meta/lib/oeqa/selftest/cases/testimage.py

diff --git a/meta/classes-recipe/testimage.bbclass 
b/meta/classes-recipe/testimage.bbclass
index b23340420f..67f7f3629e 100644
--- a/meta/classes-recipe/testimage.bbclass
+++ b/meta/classes-recipe/testimage.bbclass
@@ -118,6 +118,7 @@ python do_write_image_test_data() {
 os.remove(testdata_link)
 os.symlink(os.path.basename(testdata_name), testdata_link)
 }
+do_write_image_test_data[nostamp] = "1"
 do_write_image_test_data[vardepsexclude] += "TOPDIR"
 addtask write_image_test_data after do_rootfs before do_image_complete
 
diff --git a/meta/lib/oeqa/selftest/cases/testimage.py 
b/meta/lib/oeqa/selftest/cases/testimage.py
new file mode 100644
index 00..3ba9184821
--- /dev/null
+++ b/meta/lib/oeqa/selftest/cases/testimage.py
@@ -0,0 +1,26 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+import json
+from oeqa.selftest.case import OESelftestTestCase
+from oeqa.utils.commands import bitbake, get_bb_vars
+
+class TestDataTests(OESelftestTestCase):
+def test_regenerate(self):
+"""
+Test that variables changes are reflected in testdata.json
+"""
+test_image = "core-image-minimal"
+self.write_config('IMAGE_CLASSES += "testimage"\nTEST_VARIABLE = 
"VALUE1"')
+bitbake(test_image)
+self.append_config('TEST_VARIABLE = "VALUE2"')
+bitbake(test_image)
+
+vars = get_bb_vars(('DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'), test_image)
+testdata_json = "%s/%s.testdata.json" % (vars['DEPLOY_DIR_IMAGE'], 
vars['IMAGE_LINK_NAME'])
+with open(testdata_json, 'r') as tf:
+testdata_vars = json.load(tf)
+self.assertEqual(testdata_vars['TEST_VARIABLE'], 'VALUE2')
-- 
2.34.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#182705): 
https://lists.openembedded.org/g/openembedded-core/message/182705
Mute This Topic: https://lists.openembedded.org/mt/99501790/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH 1/2] rootfs-postcommands,testimage: Move testdata generation to own task

2023-06-13 Thread Peter Hoyes 
From: Peter Hoyes 

write_image_test_data is currently a ROOTFS_POSTPROCESS_COMMAND, even
though it does not act on the rootfs. Additionally it is only required
when running testimage.

Promote it to its own task, to decouple it from rootfs generation, and
move it to testimage.bbclass

Signed-off-by: Peter Hoyes 
---
 .../rootfs-postcommands.bbclass   | 23 ---
 meta/classes-recipe/testimage.bbclass | 21 +
 2 files changed, 21 insertions(+), 23 deletions(-)

diff --git a/meta/classes-recipe/rootfs-postcommands.bbclass 
b/meta/classes-recipe/rootfs-postcommands.bbclass
index 690fa976aa..ad7e7ccd8b 100644
--- a/meta/classes-recipe/rootfs-postcommands.bbclass
+++ b/meta/classes-recipe/rootfs-postcommands.bbclass
@@ -33,9 +33,6 @@ ROOTFS_POSTPROCESS_COMMAND += 
'${@bb.utils.contains("IMAGE_FEATURES", "read-only
 # and we don't want to disable such a default that by setting a value here.
 APPEND:append = '${@bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", " 
ro", "", d)}'
 
-# Generates test data file with data store variables expanded in json format
-ROOTFS_POSTPROCESS_COMMAND += "write_image_test_data; "
-
 # Write manifest
 IMAGE_MANIFEST = "${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.manifest"
 ROOTFS_POSTUNINSTALL_COMMAND =+ "write_image_manifest ; "
@@ -368,26 +365,6 @@ rootfs_sysroot_relativelinks () {
sysroot-relativelinks.py ${SDK_OUTPUT}/${SDKTARGETSYSROOT}
 }
 
-# Generated test data json file
-python write_image_test_data() {
-from oe.data import export2json
-
-deploy_dir = d.getVar('IMGDEPLOYDIR')
-link_name = d.getVar('IMAGE_LINK_NAME')
-testdata_name = os.path.join(deploy_dir, "%s.testdata.json" % 
d.getVar('IMAGE_NAME'))
-
-searchString = "%s/"%(d.getVar("TOPDIR")).replace("//","/")
-export2json(d, testdata_name, searchString=searchString, replaceString="")
-
-if os.path.exists(testdata_name) and link_name:
-testdata_link = os.path.join(deploy_dir, "%s.testdata.json" % 
link_name)
-if testdata_link != testdata_name:
-if os.path.lexists(testdata_link):
-os.remove(testdata_link)
-os.symlink(os.path.basename(testdata_name), testdata_link)
-}
-write_image_test_data[vardepsexclude] += "TOPDIR"
-
 # Check for unsatisfied recommendations (RRECOMMENDS)
 python rootfs_log_check_recommends() {
 log_path = d.expand("${T}/log.do_rootfs")
diff --git a/meta/classes-recipe/testimage.bbclass 
b/meta/classes-recipe/testimage.bbclass
index b48cd96575..b23340420f 100644
--- a/meta/classes-recipe/testimage.bbclass
+++ b/meta/classes-recipe/testimage.bbclass
@@ -100,6 +100,27 @@ TESTIMAGE_DUMP_DIR ?= "${LOG_DIR}/runtime-hostdump/"
 
 TESTIMAGE_UPDATE_VARS ?= "DL_DIR WORKDIR DEPLOY_DIR_IMAGE IMAGE_LINK_NAME"
 
+# Generated test data json file
+python do_write_image_test_data() {
+from oe.data import export2json
+
+deploy_dir = d.getVar('IMGDEPLOYDIR')
+link_name = d.getVar('IMAGE_LINK_NAME')
+testdata_name = os.path.join(deploy_dir, "%s.testdata.json" % 
d.getVar('IMAGE_NAME'))
+
+searchString = "%s/"%(d.getVar("TOPDIR")).replace("//","/")
+export2json(d, testdata_name, searchString=searchString, replaceString="")
+
+if os.path.exists(testdata_name) and link_name:
+testdata_link = os.path.join(deploy_dir, "%s.testdata.json" % 
link_name)
+if testdata_link != testdata_name:
+if os.path.lexists(testdata_link):
+os.remove(testdata_link)
+os.symlink(os.path.basename(testdata_name), testdata_link)
+}
+do_write_image_test_data[vardepsexclude] += "TOPDIR"
+addtask write_image_test_data after do_rootfs before do_image_complete
+
 testimage_dump_target () {
 top -bn1
 ps
-- 
2.34.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#182704): 
https://lists.openembedded.org/g/openembedded-core/message/182704
Mute This Topic: https://lists.openembedded.org/mt/99501785/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] rootfs-postcommands: Set vardeps for write_image_test_data

2023-05-24 Thread Peter Hoyes 

On 23/05/2023 18:06, Ross Burton wrote:

On 17 May 2023, at 17:29, Peter Hoyes via lists.openembedded.org 
 wrote:

The testdata.json file generated as part of the rootfs postprocess
commands contains almost all Bitbake variables and is used by OEQA test
cases to inspect the build environment. However it is only regenerated
when the rootfs task is otherwise retriggered, complicating the process
of developing OEQA test cases.

I’m about to run for dinner, but would it make more sense for this function to 
be a separate task inside testimage.bbclass that happens after rootfs?  This 
will stop the explosion of variable dependencies causing rootfs to re-run.

Ross


When running testimage under bitbake, would it make sense to simply do:

--- a/meta/classes-recipe/testimage.bbclass
+++ b/meta/classes-recipe/testimage.bbclass
@@ -350,7 +350,7 @@ def testimage_main(d):
 d.getVar("TEST_SERVER_IP"), **target_kwargs)

 # test context
-    tc = OERuntimeTestContext(td, logger, target, host_dumper,
+    tc = OERuntimeTestContext(d, logger, target, host_dumper,
   image_packages, extract_dir)

 # Load tests before starting the target

(and a few other associated tidy-ups)

Peter


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#181687): 
https://lists.openembedded.org/g/openembedded-core/message/181687
Mute This Topic: https://lists.openembedded.org/mt/98951993/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH] rootfs-postcommands: Set vardeps for write_image_test_data

2023-05-17 Thread Peter Hoyes 
From: Peter Hoyes 

The testdata.json file generated as part of the rootfs postprocess
commands contains almost all Bitbake variables and is used by OEQA test
cases to inspect the build environment. However it is only regenerated
when the rootfs task is otherwise retriggered, complicating the process
of developing OEQA test cases.

Use the vardeps mechanism to add a dependency on all the generated
datastore keys. Split out exportkeys from export2json to support this.

Add a demonstrative OE selftest to rootfspostcommandstests.

Signed-off-by: Peter Hoyes 
---
 .../rootfs-postcommands.bbclass   |  3 ++-
 meta/lib/oe/data.py   |  9 +++--
 .../selftest/cases/rootfspostcommandstests.py | 19 +++
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/meta/classes-recipe/rootfs-postcommands.bbclass 
b/meta/classes-recipe/rootfs-postcommands.bbclass
index 690fa976aa..b91498c32e 100644
--- a/meta/classes-recipe/rootfs-postcommands.bbclass
+++ b/meta/classes-recipe/rootfs-postcommands.bbclass
@@ -386,7 +386,8 @@ python write_image_test_data() {
 os.remove(testdata_link)
 os.symlink(os.path.basename(testdata_name), testdata_link)
 }
-write_image_test_data[vardepsexclude] += "TOPDIR"
+write_image_test_data[vardeps] += "${@' '.join(oe.data.exportkeys(d))}"
+write_image_test_data[vardepsexclude] += "TOPDIR DATETIME BUILDNAME 
${BB_HASHCONFIG_IGNORE_VARS}"
 
 # Check for unsatisfied recommendations (RRECOMMENDS)
 python rootfs_log_check_recommends() {
diff --git a/meta/lib/oe/data.py b/meta/lib/oe/data.py
index 37121cfad2..9989230638 100644
--- a/meta/lib/oe/data.py
+++ b/meta/lib/oe/data.py
@@ -23,8 +23,7 @@ def typed_value(key, d):
 except (TypeError, ValueError) as exc:
 bb.msg.fatal("Data", "%s: %s" % (key, str(exc)))
 
-def export2json(d, json_file, expand=True, searchString="",replaceString=""):
-data2export = {}
+def exportkeys(d):
 keys2export = []
 
 for key in d.keys():
@@ -41,6 +40,12 @@ def export2json(d, json_file, expand=True, 
searchString="",replaceString=""):
 
 keys2export.append(key)
 
+return keys2export
+
+def export2json(d, json_file, expand=True, searchString="",replaceString=""):
+data2export = {}
+keys2export = exportkeys(d)
+
 for key in keys2export:
 try:
 data2export[key] = d.getVar(key, 
expand).replace(searchString,replaceString)
diff --git a/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py 
b/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py
index 44e2c09a6f..5de4ea378a 100644
--- a/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py
+++ b/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py
@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
+import json
 import os
 import oe
 import unittest
@@ -95,3 +96,21 @@ class ShadowUtilsTidyFiles(OESelftestTestCase):
 unsorted.append(file)
 if (unsorted):
 raise Exception("The following files were not sorted by ID as 
expected: %s" % unsorted)
+
+
+class TestDataTests(OESelftestTestCase):
+def test_vardeps(self):
+"""
+Test that variables changes are reflected in testdata.json
+"""
+test_image = "core-image-minimal"
+self.write_config('TEST_VARIABLE = "VALUE1"')
+bitbake(test_image)
+self.write_config('TEST_VARIABLE = "VALUE2"')
+bitbake(test_image)
+
+vars = get_bb_vars(('DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'), test_image)
+testdata_json = "%s/%s.testdata.json" % (vars['DEPLOY_DIR_IMAGE'], 
vars['IMAGE_LINK_NAME'])
+with open(testdata_json, 'r') as tf:
+testdata_vars = json.load(tf)
+self.assertEqual(testdata_vars['TEST_VARIABLE'], 'VALUE2')
-- 
2.34.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#181496): 
https://lists.openembedded.org/g/openembedded-core/message/181496
Mute This Topic: https://lists.openembedded.org/mt/98951993/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH] u-boot: Fix syntax error in ${UBOOT_ENV}.scr compilation

2021-10-19 Thread Peter Hoyes 
From: Peter Hoyes 

A previous commit (a3d3c2d4ac421a0dde2a20825eab4434f16b2452) introduced
support for compiling a U-Boot boot script, but the logic contained a
syntax error which was only visible in the build log. Fix the error by
using separate []s for each expression in the if statement.

Signed-off-by: Peter Hoyes 
---
 meta/recipes-bsp/u-boot/u-boot.inc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/recipes-bsp/u-boot/u-boot.inc 
b/meta/recipes-bsp/u-boot/u-boot.inc
index ec4a3d201d1..2d5e46f4ef1 100644
--- a/meta/recipes-bsp/u-boot/u-boot.inc
+++ b/meta/recipes-bsp/u-boot/u-boot.inc
@@ -108,7 +108,7 @@ do_compile () {
 fi
 fi
 
-if [ -n "${UBOOT_ENV}" && "${UBOOT_ENV_SUFFIX}" = "scr" ]
+if [ -n "${UBOOT_ENV}" ] && [ "${UBOOT_ENV_SUFFIX}" = "scr" ]
 then
 ${UBOOT_MKIMAGE} -C none -A ${UBOOT_ARCH} -T script -d 
${WORKDIR}/${UBOOT_ENV_SRC} ${WORKDIR}/${UBOOT_ENV_BINARY}
 fi
-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#157117): 
https://lists.openembedded.org/g/openembedded-core/message/157117
Mute This Topic: https://lists.openembedded.org/mt/86439935/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH] u-boot: Convert ${UBOOT_ENV}.cmd into ${UBOOT_ENV}.scr

2021-10-13 Thread Peter Hoyes 
From: Peter Hoyes 

 * Add extra SRC variables to uboot-config.class for source cmd file
 * Add DEPENDS on u-boot-mkimage-native if UBOOT_ENV_SUFFIX is scr
 * Compile cmd -> scr in do_compile if UBOOT_ENV_SUFFIX is scr

Signed-off-by: Peter Hoyes 
---
 meta/classes/uboot-config.bbclass  | 2 ++
 meta/recipes-bsp/u-boot/u-boot.inc | 6 ++
 2 files changed, 8 insertions(+)

diff --git a/meta/classes/uboot-config.bbclass 
b/meta/classes/uboot-config.bbclass
index 07a303d0a07..b9ad35821ac 100644
--- a/meta/classes/uboot-config.bbclass
+++ b/meta/classes/uboot-config.bbclass
@@ -53,6 +53,8 @@ SPL_SYMLINK ?= 
"${SPL_BINARYNAME}-${MACHINE}${SPL_DELIMITER}${SPL_SUFFIX}"
 # include it in the SRC_URI and set the UBOOT_ENV parameter.
 UBOOT_ENV_SUFFIX ?= "txt"
 UBOOT_ENV ?= ""
+UBOOT_ENV_SRC_SUFFIX ?= "cmd"
+UBOOT_ENV_SRC ?= "${UBOOT_ENV}.${UBOOT_ENV_SRC_SUFFIX}"
 UBOOT_ENV_BINARY ?= "${UBOOT_ENV}.${UBOOT_ENV_SUFFIX}"
 UBOOT_ENV_IMAGE ?= "${UBOOT_ENV}-${MACHINE}-${PV}-${PR}.${UBOOT_ENV_SUFFIX}"
 UBOOT_ENV_SYMLINK ?= "${UBOOT_ENV}-${MACHINE}.${UBOOT_ENV_SUFFIX}"
diff --git a/meta/recipes-bsp/u-boot/u-boot.inc 
b/meta/recipes-bsp/u-boot/u-boot.inc
index f27fdd5998e..ec4a3d201d1 100644
--- a/meta/recipes-bsp/u-boot/u-boot.inc
+++ b/meta/recipes-bsp/u-boot/u-boot.inc
@@ -4,6 +4,7 @@ PROVIDES = "virtual/bootloader"
 PACKAGE_ARCH = "${MACHINE_ARCH}"
 
 DEPENDS += "kern-tools-native"
+DEPENDS += "${@bb.utils.contains('UBOOT_ENV_SUFFIX', 'scr', 
'u-boot-mkimage-native', '', d)}"
 
 inherit uboot-config uboot-extlinux-config uboot-sign deploy cml1 
python3native kernel-arch
 
@@ -106,6 +107,11 @@ do_compile () {
 oe_runmake -C ${S} O=${B} u-boot-initial-env
 fi
 fi
+
+if [ -n "${UBOOT_ENV}" && "${UBOOT_ENV_SUFFIX}" = "scr" ]
+then
+${UBOOT_MKIMAGE} -C none -A ${UBOOT_ARCH} -T script -d 
${WORKDIR}/${UBOOT_ENV_SRC} ${WORKDIR}/${UBOOT_ENV_BINARY}
+fi
 }
 
 do_install () {
-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156914): 
https://lists.openembedded.org/g/openembedded-core/message/156914
Mute This Topic: https://lists.openembedded.org/mt/86289690/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-