Re: [OE-core] [master][PATCH 2/5] hash equivalency and pr service
On Tue, 2020-08-25 at 09:16 -0500, Mark Hatle wrote: > On 8/25/20 7:50 AM, Joshua Watt wrote: > > On Mon, Aug 24, 2020 at 6:29 PM Mark Hatle > > wrote: > > > +PKGDATA_VARS_NOHASH = "EXTENDPRAUTO" > > > > There are so many hashes to keep track of, can we make it more > > specific than just "HASH"? How about "PKGDATA_VARS_NOOUTHASH" ? > > After talking with RP, I'm getting rid of the oe_nohash file > completely. It's > not needed any longer (after the second part of the patch.) > > We still need a way to do this clearing of the variable and such -- > and I'm not > particular about the name.. but NOTOUTHASH doesn't seem to be any > better to me > then NOHASH in this case. Except we do call this "outhash" in various places... Cheers, Richard -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#141832): https://lists.openembedded.org/g/openembedded-core/message/141832 Mute This Topic: https://lists.openembedded.org/mt/76396877/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [master][PATCH 2/5] hash equivalency and pr service
On 8/25/20 7:50 AM, Joshua Watt wrote:
> On Mon, Aug 24, 2020 at 6:29 PM Mark Hatle
> wrote:
>>
>> When the PR service is enabled a number of small changes happen to variables.
>>
>> During the do_package task, EXTENDPRAUTO will get a value placed into it that
>> is intended to be used to extend the PR. This value will get written to
>> various intermediate files and will result in the computed hash never being
>> equivalent.
>>
>> To resolve this issue, we create a new file with the extension ".oe_nohash".
>> This extension contains various values that need to be passed from task to
>> task, but NOT calculated within the scope of the task hash. The items
>> placed into the nohash are captured in package.bbclass in PKGDATA_VAR_NOHASH.
>>
>> The PKGDATA_VAR_NOHASH is also used to prevent expansion of the value when
>> users are written to the various pkgdata files. This expansion is prevented
>> by removeing the variables, resulting in the system printing the literal
>> ${VAR} when used.
>>
>> Additionally package dependencies rely on variables that also eventually
>> include EXTENDPRAUTO. So instead of resolving the variable at getVar time,
>> we keep them as references by using the same technique as above.
>>
>> Due to this change, buildhistory needs a few minor modifications to
>> know how to deal with EXTENDPRAUTO not always being available for comparison.
>>
>> Signed-off-by: Mark Hatle
>> ---
>> meta/classes/buildhistory.bbclass | 29 ++
>> meta/classes/package.bbclass | 63 ---
>> meta/lib/oe/packagedata.py| 8 +++-
>> meta/lib/oe/sstatesig.py | 2 +
>> 4 files changed, 80 insertions(+), 22 deletions(-)
>>
>> diff --git a/meta/classes/buildhistory.bbclass
>> b/meta/classes/buildhistory.bbclass
>> index 805e976ac5..2bccdfc953 100644
>> --- a/meta/classes/buildhistory.bbclass
>> +++ b/meta/classes/buildhistory.bbclass
>> @@ -99,6 +99,7 @@ python buildhistory_emit_pkghistory() {
>> import json
>> import shlex
>> import errno
>> +import oe.packagedata
>>
>> pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
>> oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE')
>> @@ -127,6 +128,7 @@ python buildhistory_emit_pkghistory() {
>> self.pkge = ""
>> self.pkgv = ""
>> self.pkgr = ""
>> +self.extendprauto = ""
>> self.size = 0
>> self.depends = ""
>> self.rprovides = ""
>> @@ -163,6 +165,8 @@ python buildhistory_emit_pkghistory() {
>> pkginfo.pkgv = value
>> elif name == "PKGR":
>> pkginfo.pkgr = value
>> +elif name == "EXTENDPRAUTO":
>> +pkginfo.extendprauto = value
>> elif name == "RPROVIDES":
>> pkginfo.rprovides = value
>> elif name == "RDEPENDS":
>> @@ -260,18 +264,24 @@ python buildhistory_emit_pkghistory() {
>>
>> pkgdest = d.getVar('PKGDEST')
>> for pkg in packagelist:
>> -pkgdata = {}
>> -with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
>> -for line in f.readlines():
>> -item = line.rstrip('\n').split(': ', 1)
>> -key = item[0]
>> -if key.endswith('_' + pkg):
>> -key = key[:-len(pkg)-1]
>> -pkgdata[key] =
>> item[1].encode('latin-1').decode('unicode_escape')
>> +pkgdata = oe.packagedata.read_pkgdatafile(os.path.join(pkgdata_dir,
>> 'runtime', pkg))
>> +
>> +npkgdata = {}
>> +for key in pkgdata.keys():
>> +if key.endswith('_' + pkg):
>> +nkey = key[:-len(pkg)-1]
>> +npkgdata[nkey] = pkgdata[key]
>> +
>> +for key in npkgdata.keys():
>> + pkgdata[key] = npkgdata[key]
>>
>> pkge = pkgdata.get('PKGE', '0')
>> pkgv = pkgdata['PKGV']
>> pkgr = pkgdata['PKGR']
>> +try:
>> +extendprauto = pkgdata['EXTENDPRAUTO']
>> +except IndexError:
>> +extendprauto = d.getVar('EXTENDPRAUTO')
>> #
>> # Find out what the last version was
>> # Make sure the version did not decrease
>> @@ -295,6 +305,7 @@ python buildhistory_emit_pkghistory() {
>> pkginfo.pkge = pkge
>> pkginfo.pkgv = pkgv
>> pkginfo.pkgr = pkgr
>> +pkginfo.extendprauto = extendprauto
>> pkginfo.rprovides =
>> sortpkglist(oe.utils.squashspaces(pkgdata.get('RPROVIDES', "")))
>> pkginfo.rdepends =
>> sortpkglist(oe.utils.squashspaces(pkgdata.get('RDEPENDS', "")))
>> pkginfo.rrecommends =
>> sortpkglist(oe.utils.squashspaces(pkgdata.get('RRECOMMENDS', "")))
>> @@ -398,6 +409,8 @@ def write_pkghistory(pkginfo, d):
>> f.write(u"PKGV = %s\n" % pkginfo.pkgv)
>> if pkginfo.pkgr != pkginfo.pr:
>> f.write(u"PKGR = %s\n" % pkgin
Re: [OE-core] [master][PATCH 2/5] hash equivalency and pr service
On Mon, Aug 24, 2020 at 6:29 PM Mark Hatle
wrote:
>
> When the PR service is enabled a number of small changes happen to variables.
>
> During the do_package task, EXTENDPRAUTO will get a value placed into it that
> is intended to be used to extend the PR. This value will get written to
> various intermediate files and will result in the computed hash never being
> equivalent.
>
> To resolve this issue, we create a new file with the extension ".oe_nohash".
> This extension contains various values that need to be passed from task to
> task, but NOT calculated within the scope of the task hash. The items
> placed into the nohash are captured in package.bbclass in PKGDATA_VAR_NOHASH.
>
> The PKGDATA_VAR_NOHASH is also used to prevent expansion of the value when
> users are written to the various pkgdata files. This expansion is prevented
> by removeing the variables, resulting in the system printing the literal
> ${VAR} when used.
>
> Additionally package dependencies rely on variables that also eventually
> include EXTENDPRAUTO. So instead of resolving the variable at getVar time,
> we keep them as references by using the same technique as above.
>
> Due to this change, buildhistory needs a few minor modifications to
> know how to deal with EXTENDPRAUTO not always being available for comparison.
>
> Signed-off-by: Mark Hatle
> ---
> meta/classes/buildhistory.bbclass | 29 ++
> meta/classes/package.bbclass | 63 ---
> meta/lib/oe/packagedata.py| 8 +++-
> meta/lib/oe/sstatesig.py | 2 +
> 4 files changed, 80 insertions(+), 22 deletions(-)
>
> diff --git a/meta/classes/buildhistory.bbclass
> b/meta/classes/buildhistory.bbclass
> index 805e976ac5..2bccdfc953 100644
> --- a/meta/classes/buildhistory.bbclass
> +++ b/meta/classes/buildhistory.bbclass
> @@ -99,6 +99,7 @@ python buildhistory_emit_pkghistory() {
> import json
> import shlex
> import errno
> +import oe.packagedata
>
> pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
> oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE')
> @@ -127,6 +128,7 @@ python buildhistory_emit_pkghistory() {
> self.pkge = ""
> self.pkgv = ""
> self.pkgr = ""
> +self.extendprauto = ""
> self.size = 0
> self.depends = ""
> self.rprovides = ""
> @@ -163,6 +165,8 @@ python buildhistory_emit_pkghistory() {
> pkginfo.pkgv = value
> elif name == "PKGR":
> pkginfo.pkgr = value
> +elif name == "EXTENDPRAUTO":
> +pkginfo.extendprauto = value
> elif name == "RPROVIDES":
> pkginfo.rprovides = value
> elif name == "RDEPENDS":
> @@ -260,18 +264,24 @@ python buildhistory_emit_pkghistory() {
>
> pkgdest = d.getVar('PKGDEST')
> for pkg in packagelist:
> -pkgdata = {}
> -with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
> -for line in f.readlines():
> -item = line.rstrip('\n').split(': ', 1)
> -key = item[0]
> -if key.endswith('_' + pkg):
> -key = key[:-len(pkg)-1]
> -pkgdata[key] =
> item[1].encode('latin-1').decode('unicode_escape')
> +pkgdata = oe.packagedata.read_pkgdatafile(os.path.join(pkgdata_dir,
> 'runtime', pkg))
> +
> +npkgdata = {}
> +for key in pkgdata.keys():
> +if key.endswith('_' + pkg):
> +nkey = key[:-len(pkg)-1]
> +npkgdata[nkey] = pkgdata[key]
> +
> +for key in npkgdata.keys():
> + pkgdata[key] = npkgdata[key]
>
> pkge = pkgdata.get('PKGE', '0')
> pkgv = pkgdata['PKGV']
> pkgr = pkgdata['PKGR']
> +try:
> +extendprauto = pkgdata['EXTENDPRAUTO']
> +except IndexError:
> +extendprauto = d.getVar('EXTENDPRAUTO')
> #
> # Find out what the last version was
> # Make sure the version did not decrease
> @@ -295,6 +305,7 @@ python buildhistory_emit_pkghistory() {
> pkginfo.pkge = pkge
> pkginfo.pkgv = pkgv
> pkginfo.pkgr = pkgr
> +pkginfo.extendprauto = extendprauto
> pkginfo.rprovides =
> sortpkglist(oe.utils.squashspaces(pkgdata.get('RPROVIDES', "")))
> pkginfo.rdepends =
> sortpkglist(oe.utils.squashspaces(pkgdata.get('RDEPENDS', "")))
> pkginfo.rrecommends =
> sortpkglist(oe.utils.squashspaces(pkgdata.get('RRECOMMENDS', "")))
> @@ -398,6 +409,8 @@ def write_pkghistory(pkginfo, d):
> f.write(u"PKGV = %s\n" % pkginfo.pkgv)
> if pkginfo.pkgr != pkginfo.pr:
> f.write(u"PKGR = %s\n" % pkginfo.pkgr)
> +if pkginfo.extendprauto:
> +f.write(u"EXTENDPRAUTO = %s\n" % pkginfo.extendprauto)
> f.write(u"RPROVIDES = %s\n" % p
[OE-core] [master][PATCH 2/5] hash equivalency and pr service
When the PR service is enabled a number of small changes happen to variables.
During the do_package task, EXTENDPRAUTO will get a value placed into it that
is intended to be used to extend the PR. This value will get written to
various intermediate files and will result in the computed hash never being
equivalent.
To resolve this issue, we create a new file with the extension ".oe_nohash".
This extension contains various values that need to be passed from task to
task, but NOT calculated within the scope of the task hash. The items
placed into the nohash are captured in package.bbclass in PKGDATA_VAR_NOHASH.
The PKGDATA_VAR_NOHASH is also used to prevent expansion of the value when
users are written to the various pkgdata files. This expansion is prevented
by removeing the variables, resulting in the system printing the literal
${VAR} when used.
Additionally package dependencies rely on variables that also eventually
include EXTENDPRAUTO. So instead of resolving the variable at getVar time,
we keep them as references by using the same technique as above.
Due to this change, buildhistory needs a few minor modifications to
know how to deal with EXTENDPRAUTO not always being available for comparison.
Signed-off-by: Mark Hatle
---
meta/classes/buildhistory.bbclass | 29 ++
meta/classes/package.bbclass | 63 ---
meta/lib/oe/packagedata.py| 8 +++-
meta/lib/oe/sstatesig.py | 2 +
4 files changed, 80 insertions(+), 22 deletions(-)
diff --git a/meta/classes/buildhistory.bbclass
b/meta/classes/buildhistory.bbclass
index 805e976ac5..2bccdfc953 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -99,6 +99,7 @@ python buildhistory_emit_pkghistory() {
import json
import shlex
import errno
+import oe.packagedata
pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE')
@@ -127,6 +128,7 @@ python buildhistory_emit_pkghistory() {
self.pkge = ""
self.pkgv = ""
self.pkgr = ""
+self.extendprauto = ""
self.size = 0
self.depends = ""
self.rprovides = ""
@@ -163,6 +165,8 @@ python buildhistory_emit_pkghistory() {
pkginfo.pkgv = value
elif name == "PKGR":
pkginfo.pkgr = value
+elif name == "EXTENDPRAUTO":
+pkginfo.extendprauto = value
elif name == "RPROVIDES":
pkginfo.rprovides = value
elif name == "RDEPENDS":
@@ -260,18 +264,24 @@ python buildhistory_emit_pkghistory() {
pkgdest = d.getVar('PKGDEST')
for pkg in packagelist:
-pkgdata = {}
-with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
-for line in f.readlines():
-item = line.rstrip('\n').split(': ', 1)
-key = item[0]
-if key.endswith('_' + pkg):
-key = key[:-len(pkg)-1]
-pkgdata[key] =
item[1].encode('latin-1').decode('unicode_escape')
+pkgdata = oe.packagedata.read_pkgdatafile(os.path.join(pkgdata_dir,
'runtime', pkg))
+
+npkgdata = {}
+for key in pkgdata.keys():
+if key.endswith('_' + pkg):
+nkey = key[:-len(pkg)-1]
+npkgdata[nkey] = pkgdata[key]
+
+for key in npkgdata.keys():
+ pkgdata[key] = npkgdata[key]
pkge = pkgdata.get('PKGE', '0')
pkgv = pkgdata['PKGV']
pkgr = pkgdata['PKGR']
+try:
+extendprauto = pkgdata['EXTENDPRAUTO']
+except IndexError:
+extendprauto = d.getVar('EXTENDPRAUTO')
#
# Find out what the last version was
# Make sure the version did not decrease
@@ -295,6 +305,7 @@ python buildhistory_emit_pkghistory() {
pkginfo.pkge = pkge
pkginfo.pkgv = pkgv
pkginfo.pkgr = pkgr
+pkginfo.extendprauto = extendprauto
pkginfo.rprovides =
sortpkglist(oe.utils.squashspaces(pkgdata.get('RPROVIDES', "")))
pkginfo.rdepends =
sortpkglist(oe.utils.squashspaces(pkgdata.get('RDEPENDS', "")))
pkginfo.rrecommends =
sortpkglist(oe.utils.squashspaces(pkgdata.get('RRECOMMENDS', "")))
@@ -398,6 +409,8 @@ def write_pkghistory(pkginfo, d):
f.write(u"PKGV = %s\n" % pkginfo.pkgv)
if pkginfo.pkgr != pkginfo.pr:
f.write(u"PKGR = %s\n" % pkginfo.pkgr)
+if pkginfo.extendprauto:
+f.write(u"EXTENDPRAUTO = %s\n" % pkginfo.extendprauto)
f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides)
f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends)
f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 7a36262eb6..5fa96eb331 100644
--- a/meta/classes/pack
