Re: [OE-core] [scarthgap 2/2] gcc: Fix for CVE-2024-0151

[Mark Hatle]

I just realized that somehow I entered the wrong CVE in the body of the patch 
itself.


This IS CVE-2024-0151, but somehow I entered CVE-2023-4039 is the patch body.

Steve can you fix this as part of the merge, or do you want me to send a V2 or a 
follow on to fix this?


--Mark

On 5/24/24 3:12 PM, Mark Hatle via lists.openembedded.org wrote:

Fix for insufficient argument checking in Secure state Entry functions
in software using Cortex-M Security Extensions (CMSE), that has been
compiled using toolchains that implement 'Arm v8-M Security Extensions
Requirements on Development Tools' prior to version 1.4, allows an
attacker to pass values to Secure state that are out of range for types
smaller than 32-bits. Out of range values might lead to incorrect
operations in secure state.

Signed-off-by: Mark Hatle 
---
  meta/recipes-devtools/gcc/gcc-13.2.inc|   1 +
  .../gcc/gcc/CVE-2024-0151.patch   | 315 ++
  2 files changed, 316 insertions(+)
  create mode 100644 meta/recipes-devtools/gcc/gcc/CVE-2024-0151.patch

diff --git a/meta/recipes-devtools/gcc/gcc-13.2.inc 
b/meta/recipes-devtools/gcc/gcc-13.2.inc
index 603377a49a..abf177822b 100644
--- a/meta/recipes-devtools/gcc/gcc-13.2.inc
+++ b/meta/recipes-devtools/gcc/gcc-13.2.inc
@@ -68,6 +68,7 @@ SRC_URI = "${BASEURI} \
 file://CVE-2023-4039.patch \
 file://0026-aarch64-Fix-loose-ldpstp-check-PR111411.patch \
 file://0027-Fix-gcc-vect-module-testcases.patch \
+   file://CVE-2024-0151.patch \
  "
  SRC_URI[sha256sum] = 
"e275e76442a6067341a27f04c5c6b83d8613144004c0413528863dc6b5c743da"
  
diff --git a/meta/recipes-devtools/gcc/gcc/CVE-2024-0151.patch b/meta/recipes-devtools/gcc/gcc/CVE-2024-0151.patch

new file mode 100644
index 00..01d55b5cdb
--- /dev/null
+++ b/meta/recipes-devtools/gcc/gcc/CVE-2024-0151.patch
@@ -0,0 +1,315 @@
+arm: Zero/Sign extends for CMSE security
+
+This patch makes the following changes:
+
+1) When calling a secure function from non-secure code then any arguments
+   smaller than 32-bits that are passed in registers are zero- or 
sign-extended.
+2) After a non-secure function returns into secure code then any return value
+   smaller than 32-bits that is passed in a register is  zero- or 
sign-extended.
+
+This patch addresses the following CVE-2024-0151.
+
+gcc/ChangeLog:
+PR target/114837
+* config/arm/arm.cc (cmse_nonsecure_call_inline_register_clear):
+  Add zero/sign extend.
+(arm_expand_prologue): Add zero/sign extend.
+
+gcc/testsuite/ChangeLog:
+
+* gcc.target/arm/cmse/extend-param.c: New test.
+* gcc.target/arm/cmse/extend-return.c: New test.
+
+CVE: CVE-2023-4039
+Upstream-Status: Backport 
[https://gcc.gnu.org/pipermail/gcc-patches/2024-April/649973.html]
+Signed-off-by: Mark Hatle 
+
+diff --git a/gcc/config/arm/arm.cc b/gcc/config/arm/arm.cc
+index 
0217abc218d60956ce727e6d008d46b9176dddc5..ea0c963a4d67ecd70e1571624e84dfe46d757df9
 100644
+--- a/gcc/config/arm/arm.cc
 b/gcc/config/arm/arm.cc
+@@ -19210,6 +19210,30 @@ cmse_nonsecure_call_inline_register_clear (void)
+ end_sequence ();
+ emit_insn_before (seq, insn);
+
++/* The AAPCS requires the callee to widen integral types narrower
++   than 32 bits to the full width of the register; but when handling
++   calls to non-secure space, we cannot trust the callee to have
++   correctly done so.  So forcibly re-widen the result here.  */
++tree ret_type = TREE_TYPE (fntype);
++if ((TREE_CODE (ret_type) == INTEGER_TYPE
++|| TREE_CODE (ret_type) == ENUMERAL_TYPE
++|| TREE_CODE (ret_type) == BOOLEAN_TYPE)
++&& known_lt (GET_MODE_SIZE (TYPE_MODE (ret_type)), 4))
++  {
++machine_mode ret_mode = TYPE_MODE (ret_type);
++rtx extend;
++if (TYPE_UNSIGNED (ret_type))
++  extend = gen_rtx_ZERO_EXTEND (SImode,
++gen_rtx_REG (ret_mode, 
R0_REGNUM));
++else
++  extend = gen_rtx_SIGN_EXTEND (SImode,
++gen_rtx_REG (ret_mode, 
R0_REGNUM));
++emit_insn_after (gen_rtx_SET (gen_rtx_REG (SImode, R0_REGNUM),
++   extend), insn);
++
++  }
++
++
+ if (TARGET_HAVE_FPCXT_CMSE)
+   {
+ rtx_insn *last, *pop_insn, *after = insn;
+@@ -23652,6 +23676,51 @@ arm_expand_prologue (void)
+
+   ip_rtx = gen_rtx_REG (SImode, IP_REGNUM);
+
++  /* The AAPCS requires the callee to widen integral types narrower
++ than 32 bits to the full width of the register; but when handling
++ calls to non-secure space, we cannot trust the callee to have
++ correctly done so.  So forcibly re-widen the result here.  */
++  if (IS_CMSE_ENTRY (func_type))
++{
++  function_args_iterator args_i

[OE-core] [master][scarthgap][PATCH v2 ] binutils: Fix aarch64 disassembly abort

[Mark Hatle]

From: Mark Hatle 

Code backported from binutils development tree.

  aarch64: Remove asserts from operand qualifier decoders [PR31595]

  Given that the disassembler should never abort when decoding
  (potentially random) data, assertion statements in the
  `get_*reg_qualifier_from_value' function family prove problematic.

  ...

Signed-off-by: Mark Hatle 
---
 meta/recipes-devtools/binutils/binutils-2.42.inc   |   1 +
 ...ove-asserts-from-operand-qualifier-decode.patch | 382 +
 2 files changed, 383 insertions(+)
 create mode 100644 
meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch

diff --git a/meta/recipes-devtools/binutils/binutils-2.42.inc 
b/meta/recipes-devtools/binutils/binutils-2.42.inc
index 3b6f47d..d2f4956 100644
--- a/meta/recipes-devtools/binutils/binutils-2.42.inc
+++ b/meta/recipes-devtools/binutils/binutils-2.42.inc
@@ -36,5 +36,6 @@ SRC_URI = "\
  file://0013-Define-alignof-using-_Alignof-when-using-C11-or-newe.patch \
  file://0014-Remove-duplicate-pe-dll.o-entry-deom-targ_extra_ofil.patch \
  file://0015-gprofng-change-use-of-bignum-to-bigint.patch \
+ file://0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch \
 "
 S  = "${WORKDIR}/git"
diff --git 
a/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
 
b/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
new file mode 100644
index 000..7b52425
--- /dev/null
+++ 
b/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
@@ -0,0 +1,382 @@
+From 5b1c70bfe0d8f84dc28237d6150b7b9d57c791a8 Mon Sep 17 00:00:00 2001
+From: Victor Do Nascimento 
+Date: Tue, 16 Apr 2024 11:49:15 +0100
+Subject: [PATCH] aarch64: Remove asserts from operand qualifier decoders
+ [PR31595]
+
+Given that the disassembler should never abort when decoding
+(potentially random) data, assertion statements in the
+`get_*reg_qualifier_from_value' function family prove problematic.
+
+Consider the random 32-bit word W, encoded in a data segment and
+encountered on execution of `objdump -D '.
+
+If:
+
+  (W & ~opcode_mask) == valid instruction
+
+Then before `print_insn_aarch64_word' has a chance to report the
+instruction as potentially undefined, an attempt will be made to have
+the qualifiers for the instruction's register operands (if any)
+decoded.  If the relevant bits do not map onto a valid qualifier for
+the matched instruction-like word, an abort will be triggered and the
+execution of objdump aborted.
+
+As this scenario is perfectly feasible and, in light of the fact that
+objdump must successfully decode all sections of a given object file,
+it is not appropriate to assert in this family of functions.
+
+Therefore, we add a new pseudo-qualifier `AARCH64_OPND_QLF_ERR' for
+handling invalid qualifier-associated values and re-purpose the
+assertion conditions in qualifier-retrieving functions to be the
+predicate guarding the returning of the calculated qualifier type.
+If the predicate fails, we return this new qualifier and allow the
+caller to handle the error as appropriate.
+
+As these functions are called either from within
+`aarch64_extract_operand' or `do_special_decoding', both of which are
+expected to return non-zero values, it suffices that callers return
+zero upon encountering `AARCH64_OPND_QLF_ERR'.
+
+Ar present the error presented in the hypothetical scenario has been
+encountered in `get_sreg_qualifier_from_value', but the change is made
+to the whole family to keep the interface consistent.
+
+Bug: https://sourceware.org/PR31595
+
+Upstream-Status: Backport [commit 2601b201e95ea0edab89342ee7137c74e88a8a79]
+
+Signed-off-by: Mark Hatle 
+---
+ .../testsuite/binutils-all/aarch64/illegal.d  |  1 +
+ .../testsuite/binutils-all/aarch64/illegal.s  |  3 +
+ include/opcode/aarch64.h  |  3 +
+ opcodes/aarch64-dis.c | 98 +++
+ 4 files changed, 87 insertions(+), 18 deletions(-)
+
+diff --git a/binutils/testsuite/binutils-all/aarch64/illegal.d 
b/binutils/testsuite/binutils-all/aarch64/illegal.d
+index 4b90a1d9f39..b69318aec85 100644
+--- a/binutils/testsuite/binutils-all/aarch64/illegal.d
 b/binutils/testsuite/binutils-all/aarch64/illegal.d
+@@ -8,5 +8,6 @@ Disassembly of section \.text:
+ 
+ 0+000 <.*>:
+ [ ]+0:[   ]+68ea18cc[ ]+.inst[]+0x68ea18cc ; undefined
++[ ]+4:[   ]+9dc39839[ ]+.inst[]+0x9dc39839 ; undefined
+ #pass
+ 
+diff --git a/binutils/testsuite/binutils-all/aarch64/illegal.s 
b/binutils/testsuite/binutils-all/aarch64/illegal.s
+index 216cbe6f265..43668c6db55 100644
+--- a/binutils/testsuite/binutils-all/aarch64/illegal.s
 b/binutils/testsuite/binutils-all/aarch64/illegal.s
+@@ -4,4 +4,7 @@
+   # ldpsw   x12, x6, [x6],#-8 ; illegal because one of the dest

[OE-core] [master][scarthgap][] binutils: Fix aarch64 disassembly abort

[Mark Hatle]

From: "Mark Hatle via lists.openembedded.org" 


Code backported from binutils development tree.

  aarch64: Remove asserts from operand qualifier decoders [PR31595]

  Given that the disassembler should never abort when decoding
  (potentially random) data, assertion statements in the
  `get_*reg_qualifier_from_value' function family prove problematic.

  ...

Signed-off-by: Mark Hatle 
---
 meta/recipes-devtools/binutils/binutils-2.42.inc   |   1 +
 ...ove-asserts-from-operand-qualifier-decode.patch | 382 +
 2 files changed, 383 insertions(+)
 create mode 100644 
meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch

diff --git a/meta/recipes-devtools/binutils/binutils-2.42.inc 
b/meta/recipes-devtools/binutils/binutils-2.42.inc
index 3b6f47d..d2f4956 100644
--- a/meta/recipes-devtools/binutils/binutils-2.42.inc
+++ b/meta/recipes-devtools/binutils/binutils-2.42.inc
@@ -36,5 +36,6 @@ SRC_URI = "\
  file://0013-Define-alignof-using-_Alignof-when-using-C11-or-newe.patch \
  file://0014-Remove-duplicate-pe-dll.o-entry-deom-targ_extra_ofil.patch \
  file://0015-gprofng-change-use-of-bignum-to-bigint.patch \
+ file://0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch \
 "
 S  = "${WORKDIR}/git"
diff --git 
a/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
 
b/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
new file mode 100644
index 000..7b52425
--- /dev/null
+++ 
b/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
@@ -0,0 +1,382 @@
+From 5b1c70bfe0d8f84dc28237d6150b7b9d57c791a8 Mon Sep 17 00:00:00 2001
+From: Victor Do Nascimento 
+Date: Tue, 16 Apr 2024 11:49:15 +0100
+Subject: [PATCH] aarch64: Remove asserts from operand qualifier decoders
+ [PR31595]
+
+Given that the disassembler should never abort when decoding
+(potentially random) data, assertion statements in the
+`get_*reg_qualifier_from_value' function family prove problematic.
+
+Consider the random 32-bit word W, encoded in a data segment and
+encountered on execution of `objdump -D '.
+
+If:
+
+  (W & ~opcode_mask) == valid instruction
+
+Then before `print_insn_aarch64_word' has a chance to report the
+instruction as potentially undefined, an attempt will be made to have
+the qualifiers for the instruction's register operands (if any)
+decoded.  If the relevant bits do not map onto a valid qualifier for
+the matched instruction-like word, an abort will be triggered and the
+execution of objdump aborted.
+
+As this scenario is perfectly feasible and, in light of the fact that
+objdump must successfully decode all sections of a given object file,
+it is not appropriate to assert in this family of functions.
+
+Therefore, we add a new pseudo-qualifier `AARCH64_OPND_QLF_ERR' for
+handling invalid qualifier-associated values and re-purpose the
+assertion conditions in qualifier-retrieving functions to be the
+predicate guarding the returning of the calculated qualifier type.
+If the predicate fails, we return this new qualifier and allow the
+caller to handle the error as appropriate.
+
+As these functions are called either from within
+`aarch64_extract_operand' or `do_special_decoding', both of which are
+expected to return non-zero values, it suffices that callers return
+zero upon encountering `AARCH64_OPND_QLF_ERR'.
+
+Ar present the error presented in the hypothetical scenario has been
+encountered in `get_sreg_qualifier_from_value', but the change is made
+to the whole family to keep the interface consistent.
+
+Bug: https://sourceware.org/PR31595
+
+Upstream-Status: Backport [commit 2601b201e95ea0edab89342ee7137c74e88a8a79]
+
+Signed-off-by: Mark Hatle 
+---
+ .../testsuite/binutils-all/aarch64/illegal.d  |  1 +
+ .../testsuite/binutils-all/aarch64/illegal.s  |  3 +
+ include/opcode/aarch64.h  |  3 +
+ opcodes/aarch64-dis.c | 98 +++
+ 4 files changed, 87 insertions(+), 18 deletions(-)
+
+diff --git a/binutils/testsuite/binutils-all/aarch64/illegal.d 
b/binutils/testsuite/binutils-all/aarch64/illegal.d
+index 4b90a1d9f39..b69318aec85 100644
+--- a/binutils/testsuite/binutils-all/aarch64/illegal.d
 b/binutils/testsuite/binutils-all/aarch64/illegal.d
+@@ -8,5 +8,6 @@ Disassembly of section \.text:
+ 
+ 0+000 <.*>:
+ [ ]+0:[   ]+68ea18cc[ ]+.inst[]+0x68ea18cc ; undefined
++[ ]+4:[   ]+9dc39839[ ]+.inst[]+0x9dc39839 ; undefined
+ #pass
+ 
+diff --git a/binutils/testsuite/binutils-all/aarch64/illegal.s 
b/binutils/testsuite/binutils-all/aarch64/illegal.s
+index 216cbe6f265..43668c6db55 100644
+--- a/binutils/testsuite/binutils-all/aarch64/illegal.s
 b/binutils/testsuite/binutils-all/aarch64/illegal.s
+@@ -4,4 +4,7 @@
+   # ldpsw   x12, x6, [x

Re: [OE-core] [scarthgap 1/2] binutils: Fix aarch64 disassembly abort

[Mark Hatle]

On 5/27/24 8:50 AM, Steve Sakoman wrote:

This patch will need to be submitted and accepted for master before I
can take it for scarthgap.


I didn't submit because I thought there was a new binutils version in 
master-next.  I guess I'm wrong.


The patch should apply the same in master.

--Mark


Thanks,

Steve

On Fri, May 24, 2024 at 1:12 PM Mark Hatle via lists.openembedded.org
 wrote:


Code backported from binutils development tree.

   aarch64: Remove asserts from operand qualifier decoders [PR31595]

   Given that the disassembler should never abort when decoding
   (potentially random) data, assertion statements in the
   `get_*reg_qualifier_from_value' function family prove problematic.

   ...

Signed-off-by: Mark Hatle 
---
  .../binutils/binutils-2.42.inc|   1 +
  ...sserts-from-operand-qualifier-decode.patch | 382 ++
  2 files changed, 383 insertions(+)
  create mode 100644 
meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch

diff --git a/meta/recipes-devtools/binutils/binutils-2.42.inc 
b/meta/recipes-devtools/binutils/binutils-2.42.inc
index 3b6f47d4ce..d2f49560f3 100644
--- a/meta/recipes-devtools/binutils/binutils-2.42.inc
+++ b/meta/recipes-devtools/binutils/binutils-2.42.inc
@@ -36,5 +36,6 @@ SRC_URI = "\
   file://0013-Define-alignof-using-_Alignof-when-using-C11-or-newe.patch \
   file://0014-Remove-duplicate-pe-dll.o-entry-deom-targ_extra_ofil.patch \
   file://0015-gprofng-change-use-of-bignum-to-bigint.patch \
+ file://0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch \
  "
  S  = "${WORKDIR}/git"
diff --git 
a/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
 
b/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
new file mode 100644
index 00..7b52425a38
--- /dev/null
+++ 
b/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
@@ -0,0 +1,382 @@
+From 5b1c70bfe0d8f84dc28237d6150b7b9d57c791a8 Mon Sep 17 00:00:00 2001
+From: Victor Do Nascimento 
+Date: Tue, 16 Apr 2024 11:49:15 +0100
+Subject: [PATCH] aarch64: Remove asserts from operand qualifier decoders
+ [PR31595]
+
+Given that the disassembler should never abort when decoding
+(potentially random) data, assertion statements in the
+`get_*reg_qualifier_from_value' function family prove problematic.
+
+Consider the random 32-bit word W, encoded in a data segment and
+encountered on execution of `objdump -D '.
+
+If:
+
+  (W & ~opcode_mask) == valid instruction
+
+Then before `print_insn_aarch64_word' has a chance to report the
+instruction as potentially undefined, an attempt will be made to have
+the qualifiers for the instruction's register operands (if any)
+decoded.  If the relevant bits do not map onto a valid qualifier for
+the matched instruction-like word, an abort will be triggered and the
+execution of objdump aborted.
+
+As this scenario is perfectly feasible and, in light of the fact that
+objdump must successfully decode all sections of a given object file,
+it is not appropriate to assert in this family of functions.
+
+Therefore, we add a new pseudo-qualifier `AARCH64_OPND_QLF_ERR' for
+handling invalid qualifier-associated values and re-purpose the
+assertion conditions in qualifier-retrieving functions to be the
+predicate guarding the returning of the calculated qualifier type.
+If the predicate fails, we return this new qualifier and allow the
+caller to handle the error as appropriate.
+
+As these functions are called either from within
+`aarch64_extract_operand' or `do_special_decoding', both of which are
+expected to return non-zero values, it suffices that callers return
+zero upon encountering `AARCH64_OPND_QLF_ERR'.
+
+Ar present the error presented in the hypothetical scenario has been
+encountered in `get_sreg_qualifier_from_value', but the change is made
+to the whole family to keep the interface consistent.
+
+Bug: https://sourceware.org/PR31595
+
+Upstream-Status: Backport [commit 2601b201e95ea0edab89342ee7137c74e88a8a79]
+
+Signed-off-by: Mark Hatle 
+---
+ .../testsuite/binutils-all/aarch64/illegal.d  |  1 +
+ .../testsuite/binutils-all/aarch64/illegal.s  |  3 +
+ include/opcode/aarch64.h  |  3 +
+ opcodes/aarch64-dis.c | 98 +++
+ 4 files changed, 87 insertions(+), 18 deletions(-)
+
+diff --git a/binutils/testsuite/binutils-all/aarch64/illegal.d 
b/binutils/testsuite/binutils-all/aarch64/illegal.d
+index 4b90a1d9f39..b69318aec85 100644
+--- a/binutils/testsuite/binutils-all/aarch64/illegal.d
 b/binutils/testsuite/binutils-all/aarch64/illegal.d
+@@ -8,5 +8,6 @@ Disassembly of section \.text:
+
+ 0+000 <.*>:
+ [ ]+0:[   ]+68ea18cc[ ]+.inst[]+0x68ea18cc ; undefined
++[ ]+4:[   ]+9dc39839[ ]+.inst[  

[OE-core] [scarthgap 1/2] binutils: Fix aarch64 disassembly abort

[Mark Hatle via lists.openembedded.org]

Code backported from binutils development tree.

  aarch64: Remove asserts from operand qualifier decoders [PR31595]

  Given that the disassembler should never abort when decoding
  (potentially random) data, assertion statements in the
  `get_*reg_qualifier_from_value' function family prove problematic.

  ...

Signed-off-by: Mark Hatle 
---
 .../binutils/binutils-2.42.inc|   1 +
 ...sserts-from-operand-qualifier-decode.patch | 382 ++
 2 files changed, 383 insertions(+)
 create mode 100644 
meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch

diff --git a/meta/recipes-devtools/binutils/binutils-2.42.inc 
b/meta/recipes-devtools/binutils/binutils-2.42.inc
index 3b6f47d4ce..d2f49560f3 100644
--- a/meta/recipes-devtools/binutils/binutils-2.42.inc
+++ b/meta/recipes-devtools/binutils/binutils-2.42.inc
@@ -36,5 +36,6 @@ SRC_URI = "\
  file://0013-Define-alignof-using-_Alignof-when-using-C11-or-newe.patch \
  file://0014-Remove-duplicate-pe-dll.o-entry-deom-targ_extra_ofil.patch \
  file://0015-gprofng-change-use-of-bignum-to-bigint.patch \
+ file://0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch \
 "
 S  = "${WORKDIR}/git"
diff --git 
a/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
 
b/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
new file mode 100644
index 00..7b52425a38
--- /dev/null
+++ 
b/meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
@@ -0,0 +1,382 @@
+From 5b1c70bfe0d8f84dc28237d6150b7b9d57c791a8 Mon Sep 17 00:00:00 2001
+From: Victor Do Nascimento 
+Date: Tue, 16 Apr 2024 11:49:15 +0100
+Subject: [PATCH] aarch64: Remove asserts from operand qualifier decoders
+ [PR31595]
+
+Given that the disassembler should never abort when decoding
+(potentially random) data, assertion statements in the
+`get_*reg_qualifier_from_value' function family prove problematic.
+
+Consider the random 32-bit word W, encoded in a data segment and
+encountered on execution of `objdump -D '.
+
+If:
+
+  (W & ~opcode_mask) == valid instruction
+
+Then before `print_insn_aarch64_word' has a chance to report the
+instruction as potentially undefined, an attempt will be made to have
+the qualifiers for the instruction's register operands (if any)
+decoded.  If the relevant bits do not map onto a valid qualifier for
+the matched instruction-like word, an abort will be triggered and the
+execution of objdump aborted.
+
+As this scenario is perfectly feasible and, in light of the fact that
+objdump must successfully decode all sections of a given object file,
+it is not appropriate to assert in this family of functions.
+
+Therefore, we add a new pseudo-qualifier `AARCH64_OPND_QLF_ERR' for
+handling invalid qualifier-associated values and re-purpose the
+assertion conditions in qualifier-retrieving functions to be the
+predicate guarding the returning of the calculated qualifier type.
+If the predicate fails, we return this new qualifier and allow the
+caller to handle the error as appropriate.
+
+As these functions are called either from within
+`aarch64_extract_operand' or `do_special_decoding', both of which are
+expected to return non-zero values, it suffices that callers return
+zero upon encountering `AARCH64_OPND_QLF_ERR'.
+
+Ar present the error presented in the hypothetical scenario has been
+encountered in `get_sreg_qualifier_from_value', but the change is made
+to the whole family to keep the interface consistent.
+
+Bug: https://sourceware.org/PR31595
+
+Upstream-Status: Backport [commit 2601b201e95ea0edab89342ee7137c74e88a8a79]
+
+Signed-off-by: Mark Hatle 
+---
+ .../testsuite/binutils-all/aarch64/illegal.d  |  1 +
+ .../testsuite/binutils-all/aarch64/illegal.s  |  3 +
+ include/opcode/aarch64.h  |  3 +
+ opcodes/aarch64-dis.c | 98 +++
+ 4 files changed, 87 insertions(+), 18 deletions(-)
+
+diff --git a/binutils/testsuite/binutils-all/aarch64/illegal.d 
b/binutils/testsuite/binutils-all/aarch64/illegal.d
+index 4b90a1d9f39..b69318aec85 100644
+--- a/binutils/testsuite/binutils-all/aarch64/illegal.d
 b/binutils/testsuite/binutils-all/aarch64/illegal.d
+@@ -8,5 +8,6 @@ Disassembly of section \.text:
+ 
+ 0+000 <.*>:
+ [ ]+0:[   ]+68ea18cc[ ]+.inst[]+0x68ea18cc ; undefined
++[ ]+4:[   ]+9dc39839[ ]+.inst[]+0x9dc39839 ; undefined
+ #pass
+ 
+diff --git a/binutils/testsuite/binutils-all/aarch64/illegal.s 
b/binutils/testsuite/binutils-all/aarch64/illegal.s
+index 216cbe6f265..43668c6db55 100644
+--- a/binutils/testsuite/binutils-all/aarch64/illegal.s
 b/binutils/testsuite/binutils-all/aarch64/illegal.s
+@@ -4,4 +4,7 @@
+   # ldpsw   x12, x6, [x6],#-8 ; illegal because one of the dest regs is 
also

[OE-core] [scarthgap 2/2] gcc: Fix for CVE-2024-0151

[Mark Hatle via lists.openembedded.org]

Fix for insufficient argument checking in Secure state Entry functions
in software using Cortex-M Security Extensions (CMSE), that has been
compiled using toolchains that implement 'Arm v8-M Security Extensions
Requirements on Development Tools' prior to version 1.4, allows an
attacker to pass values to Secure state that are out of range for types
smaller than 32-bits. Out of range values might lead to incorrect
operations in secure state.

Signed-off-by: Mark Hatle 
---
 meta/recipes-devtools/gcc/gcc-13.2.inc|   1 +
 .../gcc/gcc/CVE-2024-0151.patch   | 315 ++
 2 files changed, 316 insertions(+)
 create mode 100644 meta/recipes-devtools/gcc/gcc/CVE-2024-0151.patch

diff --git a/meta/recipes-devtools/gcc/gcc-13.2.inc 
b/meta/recipes-devtools/gcc/gcc-13.2.inc
index 603377a49a..abf177822b 100644
--- a/meta/recipes-devtools/gcc/gcc-13.2.inc
+++ b/meta/recipes-devtools/gcc/gcc-13.2.inc
@@ -68,6 +68,7 @@ SRC_URI = "${BASEURI} \
file://CVE-2023-4039.patch \
file://0026-aarch64-Fix-loose-ldpstp-check-PR111411.patch \
file://0027-Fix-gcc-vect-module-testcases.patch \
+   file://CVE-2024-0151.patch \
 "
 SRC_URI[sha256sum] = 
"e275e76442a6067341a27f04c5c6b83d8613144004c0413528863dc6b5c743da"
 
diff --git a/meta/recipes-devtools/gcc/gcc/CVE-2024-0151.patch 
b/meta/recipes-devtools/gcc/gcc/CVE-2024-0151.patch
new file mode 100644
index 00..01d55b5cdb
--- /dev/null
+++ b/meta/recipes-devtools/gcc/gcc/CVE-2024-0151.patch
@@ -0,0 +1,315 @@
+arm: Zero/Sign extends for CMSE security
+
+This patch makes the following changes:
+
+1) When calling a secure function from non-secure code then any arguments
+   smaller than 32-bits that are passed in registers are zero- or 
sign-extended.
+2) After a non-secure function returns into secure code then any return value
+   smaller than 32-bits that is passed in a register is  zero- or 
sign-extended.
+
+This patch addresses the following CVE-2024-0151.
+
+gcc/ChangeLog:
+PR target/114837
+* config/arm/arm.cc (cmse_nonsecure_call_inline_register_clear):
+  Add zero/sign extend.
+(arm_expand_prologue): Add zero/sign extend.
+
+gcc/testsuite/ChangeLog:
+
+* gcc.target/arm/cmse/extend-param.c: New test.
+* gcc.target/arm/cmse/extend-return.c: New test.
+
+CVE: CVE-2023-4039
+Upstream-Status: Backport 
[https://gcc.gnu.org/pipermail/gcc-patches/2024-April/649973.html]
+Signed-off-by: Mark Hatle 
+
+diff --git a/gcc/config/arm/arm.cc b/gcc/config/arm/arm.cc
+index 
0217abc218d60956ce727e6d008d46b9176dddc5..ea0c963a4d67ecd70e1571624e84dfe46d757df9
 100644
+--- a/gcc/config/arm/arm.cc
 b/gcc/config/arm/arm.cc
+@@ -19210,6 +19210,30 @@ cmse_nonsecure_call_inline_register_clear (void)
+ end_sequence ();
+ emit_insn_before (seq, insn);
+ 
++/* The AAPCS requires the callee to widen integral types narrower
++   than 32 bits to the full width of the register; but when handling
++   calls to non-secure space, we cannot trust the callee to have
++   correctly done so.  So forcibly re-widen the result here.  */
++tree ret_type = TREE_TYPE (fntype);
++if ((TREE_CODE (ret_type) == INTEGER_TYPE
++|| TREE_CODE (ret_type) == ENUMERAL_TYPE
++|| TREE_CODE (ret_type) == BOOLEAN_TYPE)
++&& known_lt (GET_MODE_SIZE (TYPE_MODE (ret_type)), 4))
++  {
++machine_mode ret_mode = TYPE_MODE (ret_type);
++rtx extend;
++if (TYPE_UNSIGNED (ret_type))
++  extend = gen_rtx_ZERO_EXTEND (SImode,
++gen_rtx_REG (ret_mode, 
R0_REGNUM));
++else
++  extend = gen_rtx_SIGN_EXTEND (SImode,
++gen_rtx_REG (ret_mode, 
R0_REGNUM));
++emit_insn_after (gen_rtx_SET (gen_rtx_REG (SImode, R0_REGNUM),
++   extend), insn);
++
++  }
++
++
+ if (TARGET_HAVE_FPCXT_CMSE)
+   {
+ rtx_insn *last, *pop_insn, *after = insn;
+@@ -23652,6 +23676,51 @@ arm_expand_prologue (void)
+ 
+   ip_rtx = gen_rtx_REG (SImode, IP_REGNUM);
+ 
++  /* The AAPCS requires the callee to widen integral types narrower
++ than 32 bits to the full width of the register; but when handling
++ calls to non-secure space, we cannot trust the callee to have
++ correctly done so.  So forcibly re-widen the result here.  */
++  if (IS_CMSE_ENTRY (func_type))
++{
++  function_args_iterator args_iter;
++  CUMULATIVE_ARGS args_so_far_v;
++  cumulative_args_t args_so_far;
++  bool first_param = true;
++  tree arg_type;
++  tree fndecl = current_function_decl;
++  tree fntype = TREE_TYPE (fndecl);
++  arm_init_cumulative_args (_so_far_v, fntype, NULL_RTX, fndecl);
++  args_so_far = pack_cumulative_args (_so_far_v);
+

[OE-core] [scarthgap 0/2] Toolchain fixes for Scarthgap

[Mark Hatle via lists.openembedded.org]

Two different toolchain fixes for Scarthgap.

The binutils change was found in a AMD/Xilinx specific build script that
attempts to disassemble some code as part of a firmware setup.  This
cortex-a53 baremetal firmware triggered a fault fixed by the issue.

The newer versions of binutils have already accepted this change, but
I'm not exactly sure which version(s) have it other then then the
current development tree.


The GCC change only affects cortex-M.  This is simply a port of the GCC
accepted change into Yocto Project for baremetal usage.  Unfortunately
I don't have any sort of test cases to show it works or doesn't.  It does
not break any of my existing test cases related to Linux for cortex-r
baremetal.

Again this change has already been accepted to newer versions of GCC.

Mark Hatle (2):
  binutils: Fix aarch64 disassembly abort
  gcc: Fix for CVE-2024-0151

 .../binutils/binutils-2.42.inc|   1 +
 ...sserts-from-operand-qualifier-decode.patch | 382 ++
 meta/recipes-devtools/gcc/gcc-13.2.inc|   1 +
 .../gcc/gcc/CVE-2024-0151.patch   | 315 +++
 4 files changed, 699 insertions(+)
 create mode 100644 
meta/recipes-devtools/binutils/binutils/0016-aarch64-Remove-asserts-from-operand-qualifier-decode.patch
 create mode 100644 meta/recipes-devtools/gcc/gcc/CVE-2024-0151.patch

-- 
2.34.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#199860): 
https://lists.openembedded.org/g/openembedded-core/message/199860
Mute This Topic: https://lists.openembedded.org/mt/106288401/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [master][scarthgap][PATCH] sstate.bbclass: Add _SSTATE_EXCLUDEDEPS_SYSROOT to vardepsexclude

[Mark Hatle via lists.openembedded.org]

When using tinfoil to control the build, multiple commands (serially) could
trigger an error such as:

  When reparsing bb:do_package, the basehash value changed from ... to  
The metadata is not deterministic and this needs to be fixed.
  ERROR: The following commands may help:
  ERROR: $ bitbake esw-conf -cdo_package -Snone
  ERROR: Then:
  ERROR: $ bitbake esw-conf -cdo_package -Sprintdiff

However following these commands it was not able to be reproduced.  Forcing
bitbake to dump the signatures and then running bitbake-diffsigs showed
that the value of _SSTATE_EXCLUDEDEPS_SYSROOT was being set in one run, but
was blank is a different version.

Upon inspecting the code in sstate.bbclass, one usage (without the _) is
already excludes, the leading _ version is used as a cache, only if set but
is not actually required to be defined.  So ignoring the value should work
properly.

Signed-off-by: Mark Hatle 
---

I found this issue on scarthgap, but I've seen similar symptoms back to 
langdale.
Most likely it's an issue back to kirkstone, but clearly rare outside of a
specific workdlow with tinfoil vs command line bitbake!

 meta/classes-global/sstate.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes-global/sstate.bbclass 
b/meta/classes-global/sstate.bbclass
index 04539bbb99..76a7b59636 100644
--- a/meta/classes-global/sstate.bbclass
+++ b/meta/classes-global/sstate.bbclass
@@ -1115,7 +1115,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, 
currentcount=0, summary=True,
 bb.parse.siggen.checkhashes(sq_data, missed, found, d)
 
 return found
-setscene_depvalid[vardepsexclude] = "SSTATE_EXCLUDEDEPS_SYSROOT"
+setscene_depvalid[vardepsexclude] = "SSTATE_EXCLUDEDEPS_SYSROOT 
_SSTATE_EXCLUDEDEPS_SYSROOT"
 
 BB_SETSCENE_DEPVALID = "setscene_depvalid"
 
-- 
2.34.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#198886): 
https://lists.openembedded.org/g/openembedded-core/message/198886
Mute This Topic: https://lists.openembedded.org/mt/105851669/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH 36/36] xz: upgrade 5.4.6 -> 5.6.1 _WARNING_

[Mark Hatle]

I know this request is a week or so old..

But do NOT upgrade to 'xz' 5.6.0 or 5.6.1.  It has been compromised:

https://www.openwall.com/lists/oss-security/2024/03/29/4

--Mark

On 3/14/24 8:40 AM, Richard Purdie wrote:

On Wed, 2024-03-13 at 15:08 +0800, wangmy via lists.openembedded.org
wrote:

From: Wang Mingyu 

License-Update:

*COPYING:
  Add the license for the XZ logo.
  Change most public domain parts to 0BSD.
  Update COPYING about the man pages of the scripts.
*getopt.c
  MSVC: Don't #include .
  lib/getopt*.c: Include  only HAVE_CONFIG_H is defined.
  lib: Update getopt.c from Gnulib with modifications.
  lib: Silence -Wsign-conversion in getopt.c.
  Add SPDX license identifiers to GPL, LGPL, and FSFULLR files.

Changelog:
=
* liblzma: Fixed two bugs relating to GNU indirect function (IFUNC)
   with GCC.
* xz: Changed the messages for thread reduction due to memory
   constraints to only appear under the highest verbosity level.
* Build:
     - Fixed a build issue when the header file 
   was present on the system but the Landlock system calls were
   not defined in .
     - The CMake build now warns and disables NLS if both gettext
   tools and pre-created .gmo files are missing. Previously,
   this caused the CMake build to fail.
* Minor improvements to man pages.
* Minor improvements to tests.



https://autobuilder.yoctoproject.org/typhoon/#/builders/48/builds/8737

Cheers,

Richard






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#197643): 
https://lists.openembedded.org/g/openembedded-core/message/197643
Mute This Topic: https://lists.openembedded.org/mt/105226831/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core][PATCH] testimage: allow to set runqemu bootparams option

[Mark Hatle]

Will this work in the case of a system boot?  I know if we load the kernel 
directly to memory and boot, we can pass kernel command line through QEMU. 
However many boards I have worked with in the past (and still do) do a system 
level boot that starts with BIOS/Firmware, goes into a second stage loader 
(grub/u-boot) and then starts the kernel.  How would these parameters end up 
going to the kernel in that case, or are they silently ignored?  Is silently 
ignored on a system boot the behavior we want?


--Mark

On 3/1/24 9:05 AM, M. Seben via lists.openembedded.org wrote:

From: michal seben 

Allow to set runqemu bootparams option using TEST_EXTRABOOTPARAMS
variable. This option can be used .e.g. to mask service only for
testimage task.

Signed-off-by: michal seben 
---
  meta/classes-recipe/testimage.bbclass | 4 +++-
  1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/meta/classes-recipe/testimage.bbclass 
b/meta/classes-recipe/testimage.bbclass
index ad040ee8f0..6e2800e3c3 100644
--- a/meta/classes-recipe/testimage.bbclass
+++ b/meta/classes-recipe/testimage.bbclass
@@ -47,6 +47,8 @@ TESTIMAGE_FAILED_QA_ARTIFACTS += 
"${@bb.utils.contains('DISTRO_FEATURES', 'ptest
  # TEST_QEMUBOOT_TIMEOUT can be used to set the maximum time in seconds the 
launch code will wait for the login prompt.
  # TEST_OVERALL_TIMEOUT can be used to set the maximum time in seconds the 
tests will be allowed to run (defaults to no limit).
  # TEST_QEMUPARAMS can be used to pass extra parameters to qemu, e.g. "-m 
1024" for setting the amount of ram to 1 GB.
+# TEST_EXTRABOOTPARAMS can be used to set the 'bootparams' option for the 
runqemu script
+# e.g. "systemd.mask=NetworkManager-wait-online.service" to mask particular 
service
  # TEST_RUNQEMUPARAMS can be used to pass extra parameters to runqemu, e.g. 
"gl" to enable OpenGL acceleration.
  # QEMU_USE_KVM can be set to "" to disable the use of kvm (by default it is 
enabled if target_arch == build_arch or both of them are x86 archs)
  
@@ -358,7 +360,7 @@ def testimage_main(d):

  try:
  # We need to check if runqemu ends unexpectedly
  # or if the worker send us a SIGTERM
-tc.target.start(params=d.getVar("TEST_QEMUPARAMS"), 
runqemuparams=d.getVar("TEST_RUNQEMUPARAMS"))
+tc.target.start(params=d.getVar("TEST_QEMUPARAMS"), 
extra_bootparams=d.getVar("TEST_EXTRABOOTPARAMS"), 
runqemuparams=d.getVar("TEST_RUNQEMUPARAMS"))
  import threading
  try:
  threading.Timer(int(d.getVar("TEST_OVERALL_TIMEOUT")), handle_test_timeout, 
(int(d.getVar("TEST_OVERALL_TIMEOUT")),)).start()






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#196641): 
https://lists.openembedded.org/g/openembedded-core/message/196641
Mute This Topic: https://lists.openembedded.org/mt/104665175/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] core-image-full-cmdline: add package-management

[Mark Hatle]

There are plenty of systems where we don't want package-management enabled by 
default.  This will just make core-image-full-cmdline less useful there.


In the past, it's always been up to the user to enable full package-management 
in their configuration, default has been to not have it present.  Has this changed?


On 2/29/24 9:03 AM, Michael Opdenacker via lists.openembedded.org wrote:

From: Michael Opdenacker 

Add "package-management" image feature to the core-image-full-cmdline image,
to support package upgrade testing.

Signed-off-by: Michael Opdenacker 
Suggested-by: Richard Purdie 
---
  meta/recipes-extended/images/core-image-full-cmdline.bb | 2 +-
  1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/recipes-extended/images/core-image-full-cmdline.bb 
b/meta/recipes-extended/images/core-image-full-cmdline.bb
index 4e1cf58d55..b034cd0aeb 100644
--- a/meta/recipes-extended/images/core-image-full-cmdline.bb
+++ b/meta/recipes-extended/images/core-image-full-cmdline.bb
@@ -1,7 +1,7 @@
  SUMMARY = "A console-only image with more full-featured Linux system \
  functionality installed."
  
-IMAGE_FEATURES += "splash ssh-server-openssh"

+IMAGE_FEATURES += "splash ssh-server-openssh package-management"
  
  IMAGE_INSTALL = "\

  packagegroup-core-boot \






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#196445): 
https://lists.openembedded.org/g/openembedded-core/message/196445
Mute This Topic: https://lists.openembedded.org/mt/104644618/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [poky] [Openembedded-architecture] [RFC PATCH] Add genericarm64 MACHINE using upstream defconfig

[Mark Hatle]

On 2/21/24 9:06 AM, Paul Barker wrote:

On 21/02/2024 10:57, Ross Burton wrote:

From: Ross Burton 

This is a new 64-bit "generic" Arm machine, that expects the hardware to
be SystemReady IR compatible. This is slightly forward-leaning as there's
not a _lot_ of SystemReady hardware in the wild, but most modern boards
are and the number will only grow.  Also, this is the only way to have a
'generic' machine as without standardised bootloaders and firmware it
would be impossible.

The base machine configuration isn't that exciting: it's a fully featured
machine that supports most things, booting via UEFI and an initramfs.

However, the kernel is more interesting.  This RFC uses the upstream defconfig
because unlike some other platforms, the arm64 defconfig is actively
maintained with the goal of being a 'boots on most hardware' configuration.
My argument is: why would we duplicate that effort?

The "linux-yocto way" is configuration fragments and after a week of
hair-pulling I do actually have fragments that boot on a BeaglePlay, but
to say this was a tiresome and frustrating exercise would be understating it.

So, a request for comments: is it acceptable to use the upstream defconfig in
a reference BSP?  Personally I'm torn: the Yocto way is fragments not monolithic
configs, but repeating the effort to fragmentise the configuration and then
also have it sufficiently modular that it can be used in pieces - instead of
just being a large file split up into smaller files - is a lot of effort for
what might end up being minimal gain.  My fear is we end up with a fragmented
configuration that can't be easily modified without breaking some platforms,
and badly copies what the defconfig already does.


I am in favour of this - I think the "genericarm64" machine should use
the in-tree defconfig so that it can support the widest array of
hardware. If someone wants to trim down the kernel for a particular
platform then they should probably create a specific MACHINE anyway.

If we take the other approach of building up the kernel config from
fragments, how would we know that all SystemReady IR capable systems
will be supported? Yocto Project doesn't have the resources to test
every platform.


I disagree here.  I think it would be MUCH better to have a 'SystemReady IR' 
hardware configuration.  So if SystemReady IR is desired, it is something that 
anyone can enable (starting with genericarm64).  Remember the defconfig is going 
to have more then hardware configs in it.  Will it have the right systemd 
configurations?  Will is have the magic filesystem a random user wants?  Will 
avoid having some other filesystem type that another user doesn't want?


Building up the kernel, and considering SystemReady IR as a 'hardware feature', 
and then add in the additional things that are needed for whatever reason is a 
much more reasonable way to do this and make it useful to otthers.



For the Renesas RZ SoCs I work on these days, the in-tree defconfig is
the configuration we test with the mainline kernel.


AMD does the same thing, for the kernel development it makes sense.  Kernel is 
built and tested standalone from userspace.


But with that said, I think it's the wrong way to do Yocto Project development. 
Yocto Project development needs further control and the separation of hardware 
and software configurations is pretty essential to having a system that can be 
customized appropriately.


The defconfig can be used as a guide to the other configurations, but separating 
hardware and software configs is a necessary first step in my opinion.


--Mark


Thanks,






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#196005): 
https://lists.openembedded.org/g/openembedded-core/message/196005
Mute This Topic: https://lists.openembedded.org/mt/104502773/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH 1/1] qemu: Allow native and nativesdk versions on Linux older then 4.17

[Mark Hatle]

QEMU is a symptom of the real problem.

SDK_OLDEST_KERNEL is set to 3.2.0.  If that is what we keep it as, then we need 
to make sure that the nativesdk tooling works on a pre 4.17 kernel.


If that value moves to 4.17 then we can't use the same uninative / 
buildtools-tarball with older (LTS) releases.  So there is no "good" solution here.


As you said, it's RPs decision, I see three courses here:

1) Accept qemu mmap calls could break silently on pre-4.17 kernels
2) Apply this patch, allowing SDK_OLDEST_KERNEL to remain at 3.2.0
3) Bump SDK_OLDEST_KERNEL to at least 4.17

or ?

I'm not really partial to one solution or another in this case, but we do have 
an underlying issue that needs to be resolved or we're going to be chasing bug 
reports that we have no way to resolve.


--Mark

On 1/25/24 3:41 PM, Alexander Kanavin wrote:
Initial release of Ubuntu 18.04 had 4.15 alright. Later point releases offered 
much newer kernels. It’s also altogether EOL since end of may 2023.


Why should oe-core carry these? It’s RP’s call as qemu maintainer but my vote is 
a firm no, unless additional arguments are put forward.


Alex

On Thu 25. Jan 2024 at 22.04, Mark Hatle <mailto:mark.ha...@kernel.crashing.org>> wrote:


    From: Mark Hatle mailto:mark.ha...@amd.com>>

Linux kernel 4.17 introduced two new mmap flags, MAP_FIXED_NOREPLACE and
MAP_SHARED_VALIDATE.  Starting with QEMU 8.1, these flags are now used
and required for proper system operation.  In order to build and run on a
system older then 4.17, we need to emulate this new behavior.

Not having a newer kernel could result in the mmap memory being allocated
in a way that will cause failures without QEMU checking for these
conditions.  Note, memory allocation issues are rare in my experience so
this is more of a 'just-in-case' behavior.

SDK_OLDEST_KERNEL is currently set to 3.2.0, the only way this can claim
that qemu works in an SDK is by checking the return values to emulate
the expected behavior.

Signed-off-by: Mark Hatle mailto:mark.ha...@amd.com>>
Signed-off-by: Mark Hatle mailto:mark.ha...@kernel.crashing.org>>
---
  meta/recipes-devtools/qemu/qemu.inc           |  12 +
  ...round-for-missing-MAP_FIXED_NOREPLAC.patch | 286 ++
  ...round-for-missing-MAP_SHARED_VALIDAT.patch |  51 
  3 files changed, 349 insertions(+)
  create mode 100644

meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
  create mode 100644

meta/recipes-devtools/qemu/qemu/0012-linux-user-workaround-for-missing-MAP_SHARED_VALIDAT.patch

diff --git a/meta/recipes-devtools/qemu/qemu.inc
b/meta/recipes-devtools/qemu/qemu.inc
index ccb2880402..7c31a5aa83 100644
--- a/meta/recipes-devtools/qemu/qemu.inc
+++ b/meta/recipes-devtools/qemu/qemu.inc
@@ -39,6 +39,18 @@ SRC_URI = "https://download.qemu.org/${BPN}-${PV}.tar.xz
<https://download.qemu.org/$%7BBPN%7D-$%7BPV%7D.tar.xz> \
             "
  UPSTREAM_CHECK_REGEX = "qemu-(?P\d+(\.\d+)+)\.tar"

+# SDK_OLDEST_KERNEL is set below 4.17, which is the minimum version
required by QEMU >= 8.1
+# This is due to two MMAP flags being used at certain points
+SRC_URI:append:class-nativesdk = " \
+       
file://0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch \
+       
file://0012-linux-user-workaround-for-missing-MAP_SHARED_VALIDAT.patch \
+        "
+
+# Support building and using native version on pre 4.17 kernels
+SRC_URI:append:class-native = " \
+       
file://0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch \
+       
file://0012-linux-user-workaround-for-missing-MAP_SHARED_VALIDAT.patch \
+        "

  SRC_URI[sha256sum] =
"bf00d2fa12010df8b0ade93371def58e632cb32a6bfdc5f5a0ff8e6a1fb1bf32"

diff --git

a/meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
 
b/meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
new file mode 100644
index 00..8941911fb3
--- /dev/null
+++

b/meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
@@ -0,0 +1,286 @@
+From fa9bcabe2387bb230ef82d62827ad6f93b8a1e61 Mon Sep 17 00:00:00 2001
+From: Frederic Konrad mailto:fkon...@amd.com>>
+Date: Wed, 17 Jan 2024 18:15:06 +
+Subject: [PATCH 1/2] linux-user/*: workaround for missing 
MAP_FIXED_NOREPLACE
+
+QEMU v8.1.0 recently requires MAP_FIXED_NOREPLACE flags implementation for
mmap.
+
+This is missing from ubuntu 18.04, thus this patch catches the mmap calls 
which
+could use that new flag and forwards them to mmap when MAP_FIXED_NOREPLACE
+flag isn't set or emulates them

Re: [OE-core] [PATCH 1/1] qemu: Allow native and nativesdk versions on Linux older then 4.17

[Mark Hatle]

On 1/25/24 3:42 PM, Martin Jansa wrote:
I did something similar for LGE when we were using 18.04 ubuntu, but IIRC it's 
not about the kernel version, but glibc being older than 2.27.


18.04 is already unsupported for a while, I don't think oe-core should support 
unsupported host OS versions (that's why I've never send my version).


To make this change smaller you can just 
revert c42e77a90d9244c8caf76fe0e54f84200430a4e1 from qemu.


If oe-core really needs to support this I can share my version as in:
https://github.com/shr-project/meta-webosose/commit/7528c20bf3ba7576d4611f71f987a408ac8845c2
 
<https://github.com/shr-project/meta-webosose/commit/7528c20bf3ba7576d4611f71f987a408ac8845c2>


This version includes an actual check that the mmap returned a proper value. 
The version here just sets the flags to avoid the compilation issue.


Using the buildtools-tarball in your build will accomplish the same end result, 
hides the problem but doesn't verify the mmap will actually work (or error) as 
needed.


--Mark


Regards,

On Thu, Jan 25, 2024 at 10:04 PM Mark Hatle <mailto:mark.ha...@kernel.crashing.org>> wrote:


From: Mark Hatle mailto:mark.ha...@amd.com>>

Linux kernel 4.17 introduced two new mmap flags, MAP_FIXED_NOREPLACE and
MAP_SHARED_VALIDATE.  Starting with QEMU 8.1, these flags are now used
and required for proper system operation.  In order to build and run on a
system older then 4.17, we need to emulate this new behavior.

Not having a newer kernel could result in the mmap memory being allocated
in a way that will cause failures without QEMU checking for these
conditions.  Note, memory allocation issues are rare in my experience so
this is more of a 'just-in-case' behavior.

SDK_OLDEST_KERNEL is currently set to 3.2.0, the only way this can claim
that qemu works in an SDK is by checking the return values to emulate
the expected behavior.

    Signed-off-by: Mark Hatle mailto:mark.ha...@amd.com>>
    Signed-off-by: Mark Hatle mailto:mark.ha...@kernel.crashing.org>>
---
  meta/recipes-devtools/qemu/qemu.inc           |  12 +
  ...round-for-missing-MAP_FIXED_NOREPLAC.patch | 286 ++
  ...round-for-missing-MAP_SHARED_VALIDAT.patch |  51 
  3 files changed, 349 insertions(+)
  create mode 100644

meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
  create mode 100644

meta/recipes-devtools/qemu/qemu/0012-linux-user-workaround-for-missing-MAP_SHARED_VALIDAT.patch

diff --git a/meta/recipes-devtools/qemu/qemu.inc
b/meta/recipes-devtools/qemu/qemu.inc
index ccb2880402..7c31a5aa83 100644
--- a/meta/recipes-devtools/qemu/qemu.inc
+++ b/meta/recipes-devtools/qemu/qemu.inc
@@ -39,6 +39,18 @@ SRC_URI = "https://download.qemu.org/${BPN}-${PV}.tar.xz
<https://download.qemu.org/$%7BBPN%7D-$%7BPV%7D.tar.xz> \
             "
  UPSTREAM_CHECK_REGEX = "qemu-(?P\d+(\.\d+)+)\.tar"

+# SDK_OLDEST_KERNEL is set below 4.17, which is the minimum version
required by QEMU >= 8.1
+# This is due to two MMAP flags being used at certain points
+SRC_URI:append:class-nativesdk = " \
+       
file://0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch \
+       
file://0012-linux-user-workaround-for-missing-MAP_SHARED_VALIDAT.patch \
+        "
+
+# Support building and using native version on pre 4.17 kernels
+SRC_URI:append:class-native = " \
+       
file://0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch \
+       
file://0012-linux-user-workaround-for-missing-MAP_SHARED_VALIDAT.patch \
+        "

  SRC_URI[sha256sum] =
"bf00d2fa12010df8b0ade93371def58e632cb32a6bfdc5f5a0ff8e6a1fb1bf32"

diff --git

a/meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
 
b/meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
new file mode 100644
index 00..8941911fb3
--- /dev/null
+++

b/meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
@@ -0,0 +1,286 @@
+From fa9bcabe2387bb230ef82d62827ad6f93b8a1e61 Mon Sep 17 00:00:00 2001
+From: Frederic Konrad mailto:fkon...@amd.com>>
+Date: Wed, 17 Jan 2024 18:15:06 +
+Subject: [PATCH 1/2] linux-user/*: workaround for missing 
MAP_FIXED_NOREPLACE
+
+QEMU v8.1.0 recently requires MAP_FIXED_NOREPLACE flags implementation for
mmap.
+
+This is missing from ubuntu 18.04, thus this patch catches the mmap calls 
which
+could use that new flag and forwards them to mmap when MAP_FIXED_NOREPLACE
+flag isn't set or emulates them by checking the returned address w.r.t the
+requested address.
+
+Signed-off-by: Fred

[OE-core] [PATCH 0/1] Support running qemu on kernel older then 4.17

[Mark Hatle]

We were attempting to build qemu on an Ubuntu 18.04 system and ran into
an issue where certain (newer) MMAP flags were not defined.  After further
tracking it was determined that QEMU 8.1 moved forward and only supports
usage on kernel 4.17 or newer.

Using the patch included with this, you can build for an older host, but
more importantly you can build an SDK that includes QEMU that will execute
on the 'SDK_OLDEST_KERNEL'.  While I've not gone back and verified things
work on 3.2.0 system, I have verified that Ubuntu 18.04 is working for me.

(Ubuntu 18.04 is kernel 4.15)

Mark Hatle (1):
  qemu: Allow native and nativesdk versions on Linux older then 4.17

 meta/recipes-devtools/qemu/qemu.inc   |  12 +
 ...round-for-missing-MAP_FIXED_NOREPLAC.patch | 286 ++
 ...round-for-missing-MAP_SHARED_VALIDAT.patch |  51 
 3 files changed, 349 insertions(+)
 create mode 100644 
meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
 create mode 100644 
meta/recipes-devtools/qemu/qemu/0012-linux-user-workaround-for-missing-MAP_SHARED_VALIDAT.patch

-- 
2.34.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#194332): 
https://lists.openembedded.org/g/openembedded-core/message/194332
Mute This Topic: https://lists.openembedded.org/mt/103962277/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH 1/1] qemu: Allow native and nativesdk versions on Linux older then 4.17

[Mark Hatle]

From: Mark Hatle 

Linux kernel 4.17 introduced two new mmap flags, MAP_FIXED_NOREPLACE and
MAP_SHARED_VALIDATE.  Starting with QEMU 8.1, these flags are now used
and required for proper system operation.  In order to build and run on a
system older then 4.17, we need to emulate this new behavior.

Not having a newer kernel could result in the mmap memory being allocated
in a way that will cause failures without QEMU checking for these
conditions.  Note, memory allocation issues are rare in my experience so
this is more of a 'just-in-case' behavior.

SDK_OLDEST_KERNEL is currently set to 3.2.0, the only way this can claim
that qemu works in an SDK is by checking the return values to emulate
the expected behavior.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/recipes-devtools/qemu/qemu.inc   |  12 +
 ...round-for-missing-MAP_FIXED_NOREPLAC.patch | 286 ++
 ...round-for-missing-MAP_SHARED_VALIDAT.patch |  51 
 3 files changed, 349 insertions(+)
 create mode 100644 
meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
 create mode 100644 
meta/recipes-devtools/qemu/qemu/0012-linux-user-workaround-for-missing-MAP_SHARED_VALIDAT.patch

diff --git a/meta/recipes-devtools/qemu/qemu.inc 
b/meta/recipes-devtools/qemu/qemu.inc
index ccb2880402..7c31a5aa83 100644
--- a/meta/recipes-devtools/qemu/qemu.inc
+++ b/meta/recipes-devtools/qemu/qemu.inc
@@ -39,6 +39,18 @@ SRC_URI = "https://download.qemu.org/${BPN}-${PV}.tar.xz \
"
 UPSTREAM_CHECK_REGEX = "qemu-(?P\d+(\.\d+)+)\.tar"
 
+# SDK_OLDEST_KERNEL is set below 4.17, which is the minimum version required 
by QEMU >= 8.1
+# This is due to two MMAP flags being used at certain points
+SRC_URI:append:class-nativesdk = " \
+   file://0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch \
+   file://0012-linux-user-workaround-for-missing-MAP_SHARED_VALIDAT.patch \
+"
+
+# Support building and using native version on pre 4.17 kernels
+SRC_URI:append:class-native = " \
+   file://0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch \
+   file://0012-linux-user-workaround-for-missing-MAP_SHARED_VALIDAT.patch \
+"
 
 SRC_URI[sha256sum] = 
"bf00d2fa12010df8b0ade93371def58e632cb32a6bfdc5f5a0ff8e6a1fb1bf32"
 
diff --git 
a/meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
 
b/meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
new file mode 100644
index 00..8941911fb3
--- /dev/null
+++ 
b/meta/recipes-devtools/qemu/qemu/0011-linux-user-workaround-for-missing-MAP_FIXED_NOREPLAC.patch
@@ -0,0 +1,286 @@
+From fa9bcabe2387bb230ef82d62827ad6f93b8a1e61 Mon Sep 17 00:00:00 2001
+From: Frederic Konrad 
+Date: Wed, 17 Jan 2024 18:15:06 +
+Subject: [PATCH 1/2] linux-user/*: workaround for missing MAP_FIXED_NOREPLACE
+
+QEMU v8.1.0 recently requires MAP_FIXED_NOREPLACE flags implementation for 
mmap.
+
+This is missing from ubuntu 18.04, thus this patch catches the mmap calls which
+could use that new flag and forwards them to mmap when MAP_FIXED_NOREPLACE
+flag isn't set or emulates them by checking the returned address w.r.t the
+requested address.
+
+Signed-off-by: Frederic Konrad 
+Signed-off-by: Francisco Iglesias 
+
+Upstream-Status: Inappropriate [OE specific]
+
+The upstream only supports the last two major releases of an OS.  The ones
+they have declared all have kernel 4.17 or newer.
+
+See:
+https://xilinx.slack.com/archives/D04G2647CTV/p1705074697942019
+
+https://www.qemu.org/docs/master/about/build-platforms.html
+
+ The project aims to support the most recent major version at all times for up
+ to five years after its initial release. Support for the previous major
+ version will be dropped 2 years after the new major version is released or
+ when the vendor itself drops support, whichever comes first.
+
+Signed-off-by: Mark Hatle 
+---
+ linux-user/elfload.c|  7 +++--
+ linux-user/meson.build  |  1 +
+ linux-user/mmap-fixed.c | 63 +
+ linux-user/mmap-fixed.h | 39 +
+ linux-user/mmap.c   | 31 +++-
+ linux-user/syscall.c|  1 +
+ 6 files changed, 125 insertions(+), 17 deletions(-)
+ create mode 100644 linux-user/mmap-fixed.c
+ create mode 100644 linux-user/mmap-fixed.h
+
+Index: qemu-8.2.0/linux-user/elfload.c
+===
+--- qemu-8.2.0.orig/linux-user/elfload.c
 qemu-8.2.0/linux-user/elfload.c
+@@ -22,6 +22,7 @@
+ #include "qemu/error-report.h"
+ #include "target_signal.h"
+ #include "accel/tcg/debuginfo.h"
++#include "mmap-fixed.h"
+ 
+ #ifdef TARGET_ARM
+ #include "target/arm/cpu-features.h"
+@@ -2765,9 +2766,9 @@ static abi_ulong create_elf_tables(abi_u
+ static int pg

Re: [OE-core] Removing Github release SRC_URIs from oe-core recipes?

[Mark Hatle]

On 1/15/24 7:50 AM, Jasper Orschulko via lists.openembedded.org wrote:

Hi Alex,


Okay, I've read the README file in that repo, and if i understood it
right, the process is:
- run fossology
- have a human inspect the output, and correct it on a file by file
basis (tremendous waste of time and limited developer resources even
when done the 'open source way' if you ask me but whatevs)
- place the corrected output into the above repository


Correct.


Do you really really need the 'human corrected' part of all this?


Unfortunately we do need the "human corrected" part (I wish we
wouldn't). We have industry customers (and in turn our legal
department) that demand a license compliance report and clearing on a
"per-file" basis. Currently available scanning tools are unfortunately
way to unreliable for usage without any human interaction (believe me,
we tried).

Will this be stupid amount of work? Yes. Is this compliance obligations
gone mad? IMO, absolutely yes.

But unfortunately this seems to be the way the industry is heading. We
as a supplier company are getting more and more requests of this sort
from our clients (big players in the public transport and automotive
industry) who won't "play" with us unless these obligations are
fulfilled. I've also heared similar stories from other companies.
(That's what you get when you let company lawyers go wild ️)

So I'm not happy with the situation but doing the best I can, given the
requirements. In our case that means sharing our license clearings
(which we have to do in any case) with the open source community, in
the hope that other companies have similar challenges and that we can
get some crowd-sourcing going.


It will never possibly cover all of the packages you need to ship and
match all their versions.


So what we are currently striving for is covering all target relevant
packages (aka without any special suffixes) of a "basic linux build"
(aka image-core-minimal) for LTS releases (starting with kirkstone).
Additionally, meta-ossselot will have logic for re-use of other package
versions (so does osselot as a whole), limiting curation to the diff.
I also have some ideas on dealing with openembedded patches added to
point releases (currently another pain point).


One note, when I worked on this a few years ago (pre-2020) each source file was 
correlated to an entry in our SPDX database.  The licenses were manually 
reviewed and that was all attached to a file's hash.


This way it didn't matter if you were looking at an upstream tarball, an 
upstream git repository, or any other mode of getting the source code.


Tracking was effectively "human reviewed" SPDX was compared against the patched 
file manifest.  Any sources that didn't match the checksum were flagged for 
review.  A master database of checksums were also available to see if the file 
was duplicated in other repositories as well.  This also allowed comparison of 
license/copyright and possibly patent data between different projects.  (and 
ALSO allowed exceptions for generated m4, configure.in, etc files!)


This manual review work absolutely needs to be done for some parts of the world 
and some company requirements... but be careful not to make the work so complex 
it can't be completed.


--Mark


Best,
Jasper






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193686): 
https://lists.openembedded.org/g/openembedded-core/message/193686
Mute This Topic: https://lists.openembedded.org/mt/103730186/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH RFC] bitbake.conf/pseudo: Switch from exclusion list to inclusion list

[Mark Hatle]

On 12/11/23 2:35 PM, Ross Burton wrote:

On 11 Dec 2023, at 17:35, Richard Purdie via lists.openembedded.org 
 wrote:


Currently, pseudo tracks all files referenced within it's presence unless
they're listed in an exclusion list. The exclusion list has grown to be
fairly unwieldy.

This patch swaps PSEUDO_IGNORE_PATHS for PSEUDO_INCLUDE_PATHS which in
theory should be easier and more explicit to maintain.


I’ve never understood the rationale behind exclude over include (then again 
there’s lots I don’t understand), so I’m very much for this in principle.


History:

It was originally everything, because files were referenced, linked and copied 
around random parts of the system and/or build and needed preservation of 
components.  This was very early Yocto Project and before.  (Psuedo existed for 
many years before OE/YP used it!)


Then the build/work directories became more standardized, and the components 
'copying in' from elsewhere was greatly reduced... but there were points where 
reference files were still used (think .m4 files or tex templates for instance 
for the kinds of things...)  Since we didn't know what all of the reference 
points (to the host) were, we decided to be conservative to add exclusion.


So excluding paths where we could count on owners/groups (and more importantly 
permissions) being standard became the norm.  That's why the primary exclusion was:


/usr/,/etc/,/lib,/dev/,/run/

Over time though we've gotten our dependencies understood and really severely 
limited the external components we use in the build.  (including .m4, autoconf, 
tex, etc etc etc..)   So moving to an INCLUDE seems to make more sense to me. 
Since we have a clear understanding of the path structure we use


/tmp - used by the compiler (and tons of other tooling) for temporary files, we 
need this for sure.


/proc - there are referenced entries that MIGHT be needed, especially links to 
/proc/mounts (where the link inherits perms)


/dev - I'm not sure if this is needed, but things like /dev/null and /dev/shm, 
another tmpfs, mean we probably need to track this.


/run - often contains other short-term temp files, especially for things like 
trying to process pre/post package scripting on rootfs configuration.



Then the items in WORKDIR of course.  IN THE PAST, we needed source code to be 
covered both to handle symlink references, as well as specific copy functions. 
But I suspect this is not needed any longer.  The actual build directory also is 
likely not needed anymore (but had been in the past!)  Leaving us the image, 
package, rootfs, etc directories.


So I think this maps well to "how did we get here".


The one question I ask, is why we're using ',' as a separator and not ':' like 
standard Unix style paths.  But I suspect the answer is Windows level 
support  but I'd be very tempted to move it back to ':' to make this more 
standard like other path separators.  (But that can be done at another time!)



--Mark



Ross






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#192242): 
https://lists.openembedded.org/g/openembedded-core/message/192242
Mute This Topic: https://lists.openembedded.org/mt/103113368/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [Openembedded-architecture] [OE-core] Core workflow: sstate for all, bblock/bbunlock, tools for why is sstate not being reused?

[Mark Hatle]

On 11/5/23 1:43 PM, Adrian Freihofer wrote:

On Sat, 2023-11-04 at 11:09 +, Richard Purdie wrote:

On Sat, 2023-11-04 at 11:29 +0100, adrian.freiho...@gmail.com wrote:

Hi Alex, hi Richard

After some internal discussions, I would like to clarify my
previous
answers on this topic.

  * Usually there are two different workflows
     - application developers: could use an SDK with a locked
sstate-cache.
     - Yocto/BSP developers: need an unlocked SDK. They change the
recipes.
  * A locked SDK
     - can work with setscene from SSTATE_MIRRORS
     - setscene does caching in the SSTATE_DIR (no issue about that)
     - But network problems can occur during the initial build
because
   bitbake executes many independent setscene tasks. Opening so
many
   independent connections slows down the build, especially if
the
   server treats them as a denial of service attack.
     - The denial of service problem is difficult to solve because
each
   setscene task runs in its own bibtake task. Reusing a
connection to
   download multiple sstate artifacts seems almost impossible.
   This is much easier to solve with separate sstate download
script.


FWIW, we did have a similar issue with do_fetch overloading
servers/proxies/ISPs and added:

do_fetch[number_threads] = "4"

Finding the right place to put a thread limit on overall setscene
tasks
is harder but in theory possible. Or perhaps a "network capable
tasks"
thread limit?

Is the overload caused by the initial query of sstate presence, or,
does it happen when the setscene tasks themselves run?


The most extreme situation is probably bitbake --setscene-only with an
empty TMPDIR. Each of the setscene tasks establishes a new connection.
A server receives so many connections that it treats them as a denial
of service attack by throttling. A separate script would allow the same
connection to be reused to download all the required artifacts.
Limiting the number of threads does not really solve the issue because
there are still the same amount of connections which get quickly
opened.





  * An unlocked SDK
     - Tries to download the sstate cache for changed recipes and
their
   dependencies, which obviously can't work.
     - The useless download requests slow down the build
considerably and
   cause a high load on the servers without any benefit.


Is this sstate over http(s) or something else? I seem to remember you
mentioning sftp. If this were using sftp, it would be horribly slow
as
it was designed for a light overhead "does this exist?" check which
http(s) can manage well.


Yes, we are evaluating sftp. You are right, it is not optimal from a
performance point of view. For example S3 is much faster. A compromise
is to set up a limited number of parallel sftp connections. This has
worked very well so far.

The question of why we use sftp brings us to a larger topic that is
probably relevant for almost all Yocto users, but not for the Yocto
project itself: Security.

There is usually a git server infrastructure that makes it possible to
protect Git repositories with finely graded access policies. As the
sstate-cache contains the same source code, the protection concept for
the Git repositories must also be applied to the sstate-cache
artifacts.

First of all a user authentication is required for the sstate-mirror.
An obvious idea is to use the same user authentication for the sstate-
cache server as for the Git server. In addition to https, ssh is also
often used for git repositories. SSH even offers some advantages in
terms of user-friendliness and security (if a ssh agent is used). This
consideration finally leads us to use the sftp protocol for the sstate
mirror. This is also relatively easy to administer: Simply copy the
user's public ssh keys from the git server to the sftp server.


While being able to support ssh (or a related protocol) is useful, you need to 
also remember that MANY MANY organizations absolutely block SSH access through 
their firewalls.  So _requiring_ sftp would be bad.  Allowing it's usage would 
be good.


As for logging in, https is transport 'security' but not authentication without 
additional helpers.  I think it's absolutely reasonable to say https access 
either needs an external helper for authentication purposes or it's 
un-authenticated.  If you want (internal to the company) then ssh/sftp or 
similar using the ssh-agent (or similar) should be the suggested approach.


We don't want to exclude anyone, but we want to be clear on the limitations 
based on an organization's specific choice.



If one then wants to scale an sstate-cache server for many different
projects and users, one quickly wishes for an option for authorization
at artifact level. Ideally, the access rights to the source code would
be completely transferred to the associated sstate artifacts. For such
an authorization the ssate mirror server would require the SRC_URI
which was used to compile the sstate artifact. With 

Re: [OE-core] Recent failures on master depmodwrapper-cross

[Mark Hatle]

I wanted to confirm, this did fix the problem for me.  Thanks!

On 10/16/23 10:54 AM, Yoann Congal wrote:

Le lun. 16 oct. 2023 à 17:37, Mark Hatle
 a écrit :

On 10/16/23 7:17 AM, Yoann Congal wrote:

Hi Mark,

Le lun. 16 oct. 2023 à 02:40, Mark Hatle via lists.openembedded.org
 a écrit :

Running a number of builds recently, a small number of them failed.  Note we 
ran about 20 builds and only 3 failed like this, the others completed.  I'm 
wondering if maybe there is a race condition with the new code?


FYI, RP did point me to this thread (that I did miss sorry) and I
(co-author of the "new code") will send two patches to prevent the
error the happen again.
RP's hypothesis is that this is linked to S=WORKDIR recipes. Did all
your crashed build happened on "depmodwrapper-cross" (Which is
S=WORKDIR) ?


Yes, they all failed on depmodwrapper-cross.


Realy looks like the S=WORKDIR case !


Happy to test changes for you.


Thanks!

I've sent patches here  :
[PATCH 1/2] insane: skip unimplemented-ptest on S=WORKDIR recipes
https://lists.openembedded.org/g/openembedded-core/message/189301
[PATCH 2/2] insane: unimplemented-ptest: ignore source file errors
https://lists.openembedded.org/g/openembedded-core/message/189302

I've tested that the depmodwrapper-cross recipe is now skipped for the
unimplemented-ptest check.

Regards,


--Mark


Thanks!


Error:

ERROR: depmodwrapper-cross-1.0-r0 do_patch: Error executing a python function 
in exec_func_python() autogenerated:

The stack trace of python calls that resulted in this exception/failure was:
File: 'exec_func_python() autogenerated', lineno: 2, function: 
0001:
*** 0002:do_qa_patch(d)
0003:
File: 
'/scratch/jenkins-BUILDS-eSDK-dev_eSDK-eSDK-master-next-pipeline-8_ZynqMpDrFull/sources/poky/meta/classes-global/insane.bbclass',
 lineno: 1379, function: do_qa_patch
1375:elif os.path.exists(os.path.join(srcdir, "t")) and 
any(filename.endswith('.t') for filename in os.listdir(os.path.join(srcdir, 't'))):
1376:oe.qa.handle_error("unimplemented-ptest", "%s: perl Test:: based tests 
detected" % d.getVar('PN'), d)
1377:
1378:# Detect pytest-based tests
*** 1379:elif match_line_in_files(srcdir, "**/*.py", 
r'\s*(?:import\s*pytest|from\s*pytest)'):
1380:oe.qa.handle_error("unimplemented-ptest", "%s: pytest-based tests 
detected" % d.getVar('PN'), d)
1381:
1382:# Detect meson-based tests
1383:elif os.path.exists(os.path.join(srcdir, "meson.build")) and 
match_line_in_files(srcdir, "**/meson.build", r'\s*test\s*\('):
File: 
'/scratch/jenkins-BUILDS-eSDK-dev_eSDK-eSDK-master-next-pipeline-8_ZynqMpDrFull/sources/poky/meta/classes-global/insane.bbclass',
 lineno: 1357, function: match_line_in_files
1353:
###
1354:def match_line_in_files(toplevel, filename_glob, line_regex):
1355:import pathlib
1356:toppath = pathlib.Path(toplevel)
*** 1357:for entry in toppath.glob(filename_glob):
1358:try:
1359:with open(entry, 'r', encoding='utf-8', errors='ignore') 
as f:
1360:for line in f.readlines():
1361:if re.match(line_regex, line):
File: '/usr/lib/python3.10/pathlib.py', lineno: 1034, function: glob
1030:drv, root, pattern_parts = self._flavour.parse_parts((pattern,))
1031:if drv or root:
1032:raise NotImplementedError("Non-relative patterns are 
unsupported")
1033:selector = _make_selector(tuple(pattern_parts), self._flavour)
*** 1034:for p in selector.select_from(self):
1035:yield p
1036:
1037:def rglob(self, pattern):
1038:"""Recursively yield all existing files (of any kind, including
File: '/usr/lib/python3.10/pathlib.py', lineno: 493, function: _select_from
0489:yielded = set()
0490:try:
0491:successor_select = self.successor._select_from
0492:for starting_point in 
self._iterate_directories(parent_path, is_dir, scandir):
*** 0493:for p in successor_select(starting_point, is_dir, 
exists, scandir):
0494:if p not in yielded:
0495:yield p
0496:yielded.add(p)
0497:finally:
File: '/usr/lib/python3.10/pathlib.py', lineno: 440, function: _select_from
0436:_Selector.__init__(self, child_parts, flavour)
0437:
0438:def _select_from(self, parent_path, is_dir, exists, scandir):
0439:try:
*** 0440:with scandir(parent_path) as scandir_it:
0441:entries = list(scandir_it)
0442:for entry in entries:
0443:if self.dironly:
0444:try:
Exception: FileNotFoundError: [Errno 2] No such file or directory: 
'/scratch/jenkins-BUILDS-eSD

Re: [OE-core] Recent failures on master depmodwrapper-cross

[Mark Hatle]

On 10/16/23 7:17 AM, Yoann Congal wrote:

Hi Mark,

Le lun. 16 oct. 2023 à 02:40, Mark Hatle via lists.openembedded.org
 a écrit :

Running a number of builds recently, a small number of them failed.  Note we 
ran about 20 builds and only 3 failed like this, the others completed.  I'm 
wondering if maybe there is a race condition with the new code?


FYI, RP did point me to this thread (that I did miss sorry) and I
(co-author of the "new code") will send two patches to prevent the
error the happen again.
RP's hypothesis is that this is linked to S=WORKDIR recipes. Did all
your crashed build happened on "depmodwrapper-cross" (Which is
S=WORKDIR) ?


Yes, they all failed on depmodwrapper-cross.

Happy to test changes for you.

--Mark


Thanks!


Error:

ERROR: depmodwrapper-cross-1.0-r0 do_patch: Error executing a python function 
in exec_func_python() autogenerated:

The stack trace of python calls that resulted in this exception/failure was:
File: 'exec_func_python() autogenerated', lineno: 2, function: 
0001:
*** 0002:do_qa_patch(d)
0003:
File: 
'/scratch/jenkins-BUILDS-eSDK-dev_eSDK-eSDK-master-next-pipeline-8_ZynqMpDrFull/sources/poky/meta/classes-global/insane.bbclass',
 lineno: 1379, function: do_qa_patch
1375:elif os.path.exists(os.path.join(srcdir, "t")) and 
any(filename.endswith('.t') for filename in os.listdir(os.path.join(srcdir, 't'))):
1376:oe.qa.handle_error("unimplemented-ptest", "%s: perl Test:: based tests 
detected" % d.getVar('PN'), d)
1377:
1378:# Detect pytest-based tests
*** 1379:elif match_line_in_files(srcdir, "**/*.py", 
r'\s*(?:import\s*pytest|from\s*pytest)'):
1380:oe.qa.handle_error("unimplemented-ptest", "%s: pytest-based tests 
detected" % d.getVar('PN'), d)
1381:
1382:# Detect meson-based tests
1383:elif os.path.exists(os.path.join(srcdir, "meson.build")) and 
match_line_in_files(srcdir, "**/meson.build", r'\s*test\s*\('):
File: 
'/scratch/jenkins-BUILDS-eSDK-dev_eSDK-eSDK-master-next-pipeline-8_ZynqMpDrFull/sources/poky/meta/classes-global/insane.bbclass',
 lineno: 1357, function: match_line_in_files
1353:
###
1354:def match_line_in_files(toplevel, filename_glob, line_regex):
1355:import pathlib
1356:toppath = pathlib.Path(toplevel)
*** 1357:for entry in toppath.glob(filename_glob):
1358:try:
1359:with open(entry, 'r', encoding='utf-8', errors='ignore') 
as f:
1360:for line in f.readlines():
1361:if re.match(line_regex, line):
File: '/usr/lib/python3.10/pathlib.py', lineno: 1034, function: glob
1030:drv, root, pattern_parts = self._flavour.parse_parts((pattern,))
1031:if drv or root:
1032:raise NotImplementedError("Non-relative patterns are 
unsupported")
1033:selector = _make_selector(tuple(pattern_parts), self._flavour)
*** 1034:for p in selector.select_from(self):
1035:yield p
1036:
1037:def rglob(self, pattern):
1038:"""Recursively yield all existing files (of any kind, including
File: '/usr/lib/python3.10/pathlib.py', lineno: 493, function: _select_from
0489:yielded = set()
0490:try:
0491:successor_select = self.successor._select_from
0492:for starting_point in 
self._iterate_directories(parent_path, is_dir, scandir):
*** 0493:for p in successor_select(starting_point, is_dir, 
exists, scandir):
0494:if p not in yielded:
0495:yield p
0496:yielded.add(p)
0497:finally:
File: '/usr/lib/python3.10/pathlib.py', lineno: 440, function: _select_from
0436:_Selector.__init__(self, child_parts, flavour)
0437:
0438:def _select_from(self, parent_path, is_dir, exists, scandir):
0439:try:
*** 0440:with scandir(parent_path) as scandir_it:
0441:entries = list(scandir_it)
0442:for entry in entries:
0443:if self.dironly:
0444:try:
Exception: FileNotFoundError: [Errno 2] No such file or directory: 
'/scratch/jenkins-BUILDS-eSDK-dev_eSDK-eSDK-master-next-pipeline-8_ZynqMpDrFull/build/tmp/work/zynqmp_generic-xilinx-linux/depmodwrapper-cross/1.0/sstate-build-create_spdx'

ERROR: Logfile of failure stored in: 
/scratch/jenkins-BUILDS-eSDK-dev_eSDK-eSDK-master-next-pipeline-8_ZynqMpDrFull/build/tmp/work/zynqmp_generic-xilinx-linux/depmodwrapper-cross/1.0/temp/log.do_patch.1966488








-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#189299): 
https://lists.openembedded.org/g/openembedded-core/message/189299
Mute This Topic: https://lists.openembedded.org/mt/101987519/21656
Group Owner

[OE-core] Recent failures on master depmodwrapper-cross

[Mark Hatle via lists.openembedded.org]

Running a number of builds recently, a small number of them failed.  Note we 
ran about 20 builds and only 3 failed like this, the others completed.  I'm 
wondering if maybe there is a race condition with the new code?

Error:

ERROR: depmodwrapper-cross-1.0-r0 do_patch: Error executing a python function 
in exec_func_python() autogenerated:

The stack trace of python calls that resulted in this exception/failure was:
File: 'exec_func_python() autogenerated', lineno: 2, function: 
0001:
*** 0002:do_qa_patch(d)
0003:
File: 
'/scratch/jenkins-BUILDS-eSDK-dev_eSDK-eSDK-master-next-pipeline-8_ZynqMpDrFull/sources/poky/meta/classes-global/insane.bbclass',
 lineno: 1379, function: do_qa_patch
1375:elif os.path.exists(os.path.join(srcdir, "t")) and 
any(filename.endswith('.t') for filename in os.listdir(os.path.join(srcdir, 't'))):
1376:oe.qa.handle_error("unimplemented-ptest", "%s: perl Test:: based tests 
detected" % d.getVar('PN'), d)
1377:
1378:# Detect pytest-based tests
*** 1379:elif match_line_in_files(srcdir, "**/*.py", 
r'\s*(?:import\s*pytest|from\s*pytest)'):
1380:oe.qa.handle_error("unimplemented-ptest", "%s: pytest-based tests 
detected" % d.getVar('PN'), d)
1381:
1382:# Detect meson-based tests
1383:elif os.path.exists(os.path.join(srcdir, "meson.build")) and 
match_line_in_files(srcdir, "**/meson.build", r'\s*test\s*\('):
File: 
'/scratch/jenkins-BUILDS-eSDK-dev_eSDK-eSDK-master-next-pipeline-8_ZynqMpDrFull/sources/poky/meta/classes-global/insane.bbclass',
 lineno: 1357, function: match_line_in_files
1353:
###
1354:def match_line_in_files(toplevel, filename_glob, line_regex):
1355:import pathlib
1356:toppath = pathlib.Path(toplevel)
*** 1357:for entry in toppath.glob(filename_glob):
1358:try:
1359:with open(entry, 'r', encoding='utf-8', errors='ignore') 
as f:
1360:for line in f.readlines():
1361:if re.match(line_regex, line):
File: '/usr/lib/python3.10/pathlib.py', lineno: 1034, function: glob
1030:drv, root, pattern_parts = self._flavour.parse_parts((pattern,))
1031:if drv or root:
1032:raise NotImplementedError("Non-relative patterns are 
unsupported")
1033:selector = _make_selector(tuple(pattern_parts), self._flavour)
*** 1034:for p in selector.select_from(self):
1035:yield p
1036:
1037:def rglob(self, pattern):
1038:"""Recursively yield all existing files (of any kind, including
File: '/usr/lib/python3.10/pathlib.py', lineno: 493, function: _select_from
0489:yielded = set()
0490:try:
0491:successor_select = self.successor._select_from
0492:for starting_point in 
self._iterate_directories(parent_path, is_dir, scandir):
*** 0493:for p in successor_select(starting_point, is_dir, 
exists, scandir):
0494:if p not in yielded:
0495:yield p
0496:yielded.add(p)
0497:finally:
File: '/usr/lib/python3.10/pathlib.py', lineno: 440, function: _select_from
0436:_Selector.__init__(self, child_parts, flavour)
0437:
0438:def _select_from(self, parent_path, is_dir, exists, scandir):
0439:try:
*** 0440:with scandir(parent_path) as scandir_it:
0441:entries = list(scandir_it)
0442:for entry in entries:
0443:if self.dironly:
0444:try:
Exception: FileNotFoundError: [Errno 2] No such file or directory: 
'/scratch/jenkins-BUILDS-eSDK-dev_eSDK-eSDK-master-next-pipeline-8_ZynqMpDrFull/build/tmp/work/zynqmp_generic-xilinx-linux/depmodwrapper-cross/1.0/sstate-build-create_spdx'

ERROR: Logfile of failure stored in: 
/scratch/jenkins-BUILDS-eSDK-dev_eSDK-eSDK-master-next-pipeline-8_ZynqMpDrFull/build/tmp/work/zynqmp_generic-xilinx-linux/depmodwrapper-cross/1.0/temp/log.do_patch.1966488

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#189254): 
https://lists.openembedded.org/g/openembedded-core/message/189254
Mute This Topic: https://lists.openembedded.org/mt/101987519/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [Openembedded-architecture] Security processes: YP needs

[Mark Hatle]

On 9/15/23 2:59 AM, Marta Rybczynska wrote:

On Wed, Sep 13, 2023 at 6:28 PM Mark Hatle
 wrote:

* Visibility of the security work of the YP

There is much work on security in the YP, but it lacks visibility.


Is there a common nexus for this work? eg. do most of the folks who are
doing security work tend to congregate on the security sublist?


Security means different things to different people.  I.e.

1) Secure design
 - Is the system designed to have security services, if so are the defaults
setup to both be appropriate and also functional?

2) Additional security software
 - i.e. meta-security, what additional software can be available to enhance
security design/implementation of the system

3) Security (bug) response
 - This is where I see a lack of common nexus for work.  We don't have a 
good
place to discuss CVE specific information.  Now the question really is, should
we have a separate space.  CVEs are just bugs.  Bugs are usually worked on via
the main mailing list.  So that argument says no, we shouldn't have a special
list.  BUT the perception is CVEs are "special", so maybe a list specifically to
discuss the ramifications of a CVE, fix/mitigation strategy or similar would
make sense -- keeping actual bug fixes to the main mailing list?



It might e interesting to have opinion on people who are submitting CVE fixes...
Would keeping that discussion in a separate place be helpful?


Ya, a security mailing list can be interesting for those types of discussions, 
but ultimately the code needs to go to the regular pull list -- which depending 
on the project (and level of discussions) it may make sense just to have those 
discussions on the regular list.  It's tricky, and I'm not sure what the right 
answer is here.




* SRTool

We might decide to use it again. It allows one to do much but requires
constant commitment.


I think I passed over the wiki pages and presentations for SRTool once,
a while ago. But I didn't pay much attention at the time because it
wasn't clear *what it did*.

After reviewing it again, I think it might be the kind of tooling I've
been searching for to help my team coordinate our CVE response work.
I'll test it out and see if it is something I can use/contribute towards.


SR Tool requires constant feeding and maintenance from staff, at a minimum to do
initial triage work.  This means we need a small group of individuals who can
take the new items (and changes to existing) and comment on a regular (daily)
basis.  This is the part we've not been terribly successful in the past.  People
are great at delivering patches, but trying to do the proactive (before
cve-checker) evaluation of CVEs is an activity that often feels like busy work,
so it's easy to get behind on and never catch up.

I would love to see the project use SR Tool to manage CVE information, (bugzilla
is where the bugs need to be managed and processed), as well as cve-checker to
be able to continue to feed information or what we believe the current state of
things are.  This combination would give us per-emptive notification of new
items (SR-Tool), and late validation of items (cve-checker) on the perceived
state of things.


SRTools code base (https://git.yoctoproject.org/srtool) has seen no changes for
4 years. If we decide to use, we'll also need to maintain the tool. Is Windriver
going to update the fork? David (adding in copy), do you have any information?

Otherwise we would need to maintain our version, and update to the code
to take into account how the world have changed. For example, with the
  CVE v5 JSON, using the CVE database directly for the feed of new CVE list
makes more sense than using NVD, for example. For the reason of performance
and delay. When SRTool was developed, that data wasn't available.


Last time I used it was almost exactly 4 years ago.

The tool itself is pretty simple, it's the data import/export that is the 
complex bit(s).  Maybe the lesson here isn't to use SR Tool, but take some of 
the concepts from it and maybe implement something ourselves (in the future).


The key things are:

1) Automatic import from external CVE/Security sources (not all security items 
are CVEs)


2) A way to triage the information, and do LOCAL edits of the information
   - Way for the user to query what's new?
   - Way for the user to query what's changed since last time?
   - Way for the user to query other things...
   - Local edit could be YP 'status'
   - Local edit could add public OR private information about a given item
   - Local edits should be able to link a problem with a bug and release
   - Local edits should be able to TRACK progress of a bug
   - Local edits to CREATE security items not otherwise known (either YP 
specific, or based on bug reports, etc)
   - A way to temporarily set things as 'restricted', only for specific people 
to view until it's public information.


3) Way to generate reports for users.
   - General report
   - CVE Specific

Re: [OE-core] [Openembedded-architecture] Security processes: YP needs

[Mark Hatle]

On 9/13/23 11:00 AM, Alex Stewart wrote:

Thanks for driving this Marta. Internally and externally, it feels like
we're just on the cusp of everyone *suddenly caring* about our security
response strategy. So it's good to see that we're making moves in that
direction.

In general, this list looks complete to me. I'm primarily interested in
the response coordination, triage, and tracking usecases. Those are the
biggest pain points for my team, at the moment. And that is primarily
driven by a lack of tooling.

More responses inline.

On 9/13/23 07:52, Marta Rybczynska via lists.openembedded.org wrote:

[You don't often get email from rybczynska=gmail@lists.openembedded.org. 
Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]

Hello,
I've been working recently on collecting what works and what doesn't
in YP security processes. The goal is to go forward and define an
actionable strategy!

Today, I'd like to share with you the summary of what I have heard as
needs from several people (those in Cc:).

I want the community to comment and tell us what you find important
and what you'd like to see added or changed from this list.

* CVEs: Visibility if YP is vulnerable or not

People want to be able to check/look up a specific CVE; it might be a
CVE unrelated to YP
(eg. package not included, Windows issue). The cve-checker result is a
part of the solution, but people also want to know which CVEs do not
apply.


I'm not sure I understand this usecase. Is there a reason those people
can't/won't just lookup the CVE on the NIST site?


Management goes to an engineer and says "Customer XYZ says we need a statement 
if CVE-2024-12345 affects us.  Can you please comment?"


Engineer goes to the Yocto Project "list", and looks the number up and doesn't 
find it.  Does this mean we're affects?  We're not affected?  We were affected, 
but it's been fixed (if so when?), etc?


So then they have to go to NIST, look at the CVE, find the information and do 
the evaluation if Yocto Project is affected.


Instead what (I have observed) is that people who like to go to a single list 
(for Yocto Project) information, look up a CVE and get a clear statement of: 
This affects us, this does not affect us, we did not evaluate it or it was fixed 
by commit XYZ in branch


Then if the item is "not evaluated", they can THEN got to NIST for their own 
evaluation.  This saves a huge amount of time for people who are regularly 
requested to respond to these messages.



* CVEs: synchronization of the work on fixes

Currently, there is no synchronization; multiple parties might be
working on the same fix while nobody is working on another. There
might be duplication of work.
Ross has https://wiki.yoctoproject.org/wiki/CVE_Status


Has there been any discussion of adopting the OpenVEX document standard
that the Chainguard guys are putting together? [1] It seems like the VEX
use-cases align well with our needs around tracking and coordinating CVE
response between YP member and individual developers.

I've been considering it for my internal use for a while. And also
considering replacing the existing cve_check output JSON with OpenVEX,
once it has stabilized.

[1] https://github.com/openvex/spec


* Triaging of security issues

Related to CVE fixes and includes issues reported directly to the YP.
Some issues are more likely to be serious for embedded products
(attack by network), so not all has the same priority.


I'll note here that some of us are sinners and do actually support
network-attached (and internet-attached) embedded devices. :)

But the greater point of OS vendors being able to triage and assign
vendor-specific severities to incoming issues is absolutely important to
my use-cases.


* Private security communication

A way to send a notification of a non-public security issue. For
researchers, other projects etc.
The security alias exists, but only some people know about its existence.

* Visibility of the security work of the YP

There is much work on security in the YP, but it lacks visibility.


Is there a common nexus for this work? eg. do most of the folks who are
doing security work tend to congregate on the security sublist?


Security means different things to different people.  I.e.

1) Secure design
   - Is the system designed to have security services, if so are the defaults 
setup to both be appropriate and also functional?


2) Additional security software
   - i.e. meta-security, what additional software can be available to enhance 
security design/implementation of the system


3) Security (bug) response
   - This is where I see a lack of common nexus for work.  We don't have a good 
place to discuss CVE specific information.  Now the question really is, should 
we have a separate space.  CVEs are just bugs.  Bugs are usually worked on via 
the main mailing list.  So that argument says no, we shouldn't have a special 
list.  BUT the perception is CVEs are "special", so maybe a 

[OE-core] [PATCH] tcf-agent: Update to 1.8.0 release

[Mark Hatle via lists.openembedded.org]

New 1.8 release of tcf-agent.  Implements DWARF 5 support and various
bug fixes.

Changelog since last SRCREV:
   Releng: Upversion TCF to 1.8
   Fixed possible SEGFAULT after error message queue overflow
   Fixed misspelling in a comment
   TCF Agent: update breakpoint error message
   Fixed regression: possible segfault in run_safe_events()
   Bug 581978 - TCF agent wrong handle the call frame debug info generated by 
LLVM 16 for RISC-V
   DWARF: a bit faster implementation of dio_ReadAddressX()
   Bug 581971 - Failed to handle loclist for DWARF 5
   Fixed handling of situation when a context resumed or exited during 
breakpoint evaluation
   Bug 581799 - when loads .debug_info section from dwarf 5 file, the content 
of some part are zero
   Updated examples/daytime/readme.txt

Signed-off-by: Mark Hatle 
---
 meta/recipes-devtools/tcf-agent/tcf-agent_git.bb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/recipes-devtools/tcf-agent/tcf-agent_git.bb 
b/meta/recipes-devtools/tcf-agent/tcf-agent_git.bb
index 9e77f12b53..7d151d4642 100644
--- a/meta/recipes-devtools/tcf-agent/tcf-agent_git.bb
+++ b/meta/recipes-devtools/tcf-agent/tcf-agent_git.bb
@@ -6,8 +6,8 @@ BUGTRACKER = "https://bugs.eclipse.org/bugs/;
 LICENSE = "EPL-1.0 | EDL-1.0"
 LIC_FILES_CHKSUM = "file://edl-v10.html;md5=522a390a83dc186513f0500543ad3679"
 
-SRCREV = "4a2c4baaccbc8c29ce0297705de9a4e096d57ce5"
-PV = "1.7.0+git${SRCPV}"
+SRCREV = "1f11747e83ebf4f53e8d17f430136f92ec378709"
+PV = "1.8.0+git${SRCPV}"
 
 UPSTREAM_CHECK_GITTAGREGEX = "(?P(\d+(\.\d+)+))"
 SRC_URI = 
"git://git.eclipse.org/r/tcf/org.eclipse.tcf.agent.git;protocol=https;branch=master
 \
-- 
2.34.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#185624): 
https://lists.openembedded.org/g/openembedded-core/message/185624
Mute This Topic: https://lists.openembedded.org/mt/100609626/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core][langdale] oeqa/qemurunner: update exception class for QMP API changes

[Mark Hatle]

From: Ross Burton 

Signed-off-by: Ross Burton 
Signed-off-by: Alexandre Belloni 
(cherry picked from commit c1841ab1e7b4e078cea77001e83e733764bb65ea)
Signed-off-by: Mark Hatle 
---
 meta/lib/oeqa/utils/qemurunner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/utils/qemurunner.py 
b/meta/lib/oeqa/utils/qemurunner.py
index a455b3b389..ca75ce7e70 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -356,7 +356,7 @@ class QemuRunner:
 except OSError as msg:
 self.logger.warning("Failed to connect qemu monitor socket: %s 
File: %s" % (msg, msg.filename))
 return False
-except qmp.QMPConnectError as msg:
+except qmp.legacy.QMPError as msg:
 self.logger.warning("Failed to communicate with qemu monitor: 
%s" % (msg))
 return False
 finally:
-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#183162): 
https://lists.openembedded.org/g/openembedded-core/message/183162
Mute This Topic: https://lists.openembedded.org/mt/99666400/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] Langdale?

[Mark Hatle]

I know there won't be any more releases of langdale (as well as automatic 
backports for security and other things.)  However, I found a bug in the oeqa 
qemurunner (fixed in Mickledore).  Would you be willing to take the backport fix 
(one line) or is the branch closed for further development?


Thanks!
--Mark

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#183153): 
https://lists.openembedded.org/g/openembedded-core/message/183153
Mute This Topic: https://lists.openembedded.org/mt/99648680/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH][pseudo] Move __*xstat* and __xmknod functions to new subport 'old__x'

[Mark Hatle]

Ok, so adding this patch will move from the legacy behavior to the new behavior 
(not using the legacy stat interfaces).  This will only work on machines where 
the software has been built for only the newer exposed interfaces.


I am surprised Debian 11 failed, as it should have been new enough the old 
interfaces don't exist.  I don't know about debian11 or stream8.


Can you modify the patch you applied and re-run your tests?   Try this change:

In the patch: meta/recipes-devtools/pseudo/files/new_glibc.patch, go down to the 
patch chunk for ports/linux/subports and then change:


++if ${CC} -c -o dummy.o dummy.c >/dev/null 2>&1; then
++echo "linux/old__x"
++fi

to

++if true; then
++echo "linux/old__x"
++fi


The change above SHOULD restore the old compatible interface behavior to pseudo. 
 This will verify that the other changes did not introduce this fault.



Assuming this works, the intergration patch that I'll work on will be different. 
 The code check-in will be similar to the original path proposed (with the 
README updated).  A local patch to the integration will exist to make the change 
listed above for x86, x86_64 and aarch64.  This will restore legacy behavior on 
those architectures until we get to a point where the suspect interfaces are no 
longer being used.


I will also be adding a second commit that defines wrappers for the obsolete 
interfaces, so if they ARE used they will trigger an abort, so we can more 
quickly detect the machines.  Ultimately we want to stop wrapping the obsolete 
interfaces once the distributions are no longer using them, but we need a way to 
detect this.


(I would like a switch to configure to enable the obsolete interfaces, but I've 
not figured out how to pass configure options into the subports function call. 
I'll continue to look into that, but it really shouldn't be necessary outside of 
the YP use-case where we want one pseudo to run across a variety of hosts.)


--Mark

On 6/1/23 9:20 AM, Alexandre Belloni via lists.openembedded.org wrote:

On 01/06/2023 09:15:25-0500, Mark Hatle wrote:

Did you or someone else manually add this patch for testing?  I wasn't aware 
that it had gone in for any sort of testing eyt.

I'd like to see the integration used so I can understand how the test was 
performed.


Sure, I added the patch, here:

https://git.yoctoproject.org/poky-contrib/commit/?h=abelloni/master-next=9b8298bf6dc6



--Mark

On 6/1/23 5:43 AM, Alexandre Belloni wrote:

Hello Mark,

This causes failures on opensuse154 and debian11 workers. I've tried to
get pseudo.log for the failures but they are not present on debian11 and
are not interesting for opensuse154.

debian11 failures look like that:

https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/7184/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/7184/steps/13/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/7152/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/7152/steps/13/logs/stdio

opensuse:

https://autobuilder.yoctoproject.org/typhoon/#/builders/120/builds/2819/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/20/builds/7597/steps/13/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/53/builds/7211/steps/13/logs/stdio

stream8 is similar:

https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/2939/steps/13/logs/stdio


I also suspect this causes:

ERROR: cmdline-shebang-wrapper-test-1.0-r0 do_install: Wrapper permissions for 
/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-1782300/tmp/work/core2-64-poky-linux/cmdline-shebang-wrapper-test/1.0-r0/image/usr/bin/test.real
 not preserved. Found 600 but expected 400
ERROR: cmdline-shebang-wrapper-test-1.0-r0 do_install: 
ExecutionError('/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-1782300/tmp/work/core2-64-poky-linux/cmdline-shebang-wrapper-test/1.0-r0/temp/run.do_install.322689',
 1, None, None)
ERROR: Logfile of failure stored in: 
/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-1782300/tmp/work/core2-64-poky-linux/cmdline-shebang-wrapper-test/1.0-r0/temp/log.do_install.322689
NOTE: recipe cmdline-shebang-wrapper-test-1.0-r0: task do_install: Failed
ERROR: Task 
(/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-1782300/meta-selftest/recipes-test/wrapper/cmdline-shebang-wrapper-test.bb:do_install)
 failed with exit code '1'



On 30/05/2023 20:01:26-0700, Mark Hatle via lists.openembedded.org wrote:

Changes to eliminate __*.c function usage were based on the patch:

 From: JiaLing Zhang 
 Subject: [OE-core] [PATCH v4] Fixes pseudo build in loongarch64

 Fixes [YOCTO #15110]

 Some functions used in the project have been removed from glibc. After the 
removal of these functions,
 the architecture in glibc will not include the removed funct

Re: [OE-core] [PATCH][pseudo] Move __*xstat* and __xmknod functions to new subport 'old__x'

[Mark Hatle via lists.openembedded.org]

Did you or someone else manually add this patch for testing?  I wasn't aware 
that it had gone in for any sort of testing eyt.

I'd like to see the integration used so I can understand how the test was 
performed.

--Mark

On 6/1/23 5:43 AM, Alexandre Belloni wrote:

Hello Mark,

This causes failures on opensuse154 and debian11 workers. I've tried to
get pseudo.log for the failures but they are not present on debian11 and
are not interesting for opensuse154.

debian11 failures look like that:

https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/7184/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/42/builds/7184/steps/13/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/7152/steps/12/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/74/builds/7152/steps/13/logs/stdio

opensuse:

https://autobuilder.yoctoproject.org/typhoon/#/builders/120/builds/2819/steps/14/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/20/builds/7597/steps/13/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/53/builds/7211/steps/13/logs/stdio

stream8 is similar:

https://autobuilder.yoctoproject.org/typhoon/#/builders/117/builds/2939/steps/13/logs/stdio


I also suspect this causes:

ERROR: cmdline-shebang-wrapper-test-1.0-r0 do_install: Wrapper permissions for 
/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-1782300/tmp/work/core2-64-poky-linux/cmdline-shebang-wrapper-test/1.0-r0/image/usr/bin/test.real
 not preserved. Found 600 but expected 400
ERROR: cmdline-shebang-wrapper-test-1.0-r0 do_install: 
ExecutionError('/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-1782300/tmp/work/core2-64-poky-linux/cmdline-shebang-wrapper-test/1.0-r0/temp/run.do_install.322689',
 1, None, None)
ERROR: Logfile of failure stored in: 
/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-1782300/tmp/work/core2-64-poky-linux/cmdline-shebang-wrapper-test/1.0-r0/temp/log.do_install.322689
NOTE: recipe cmdline-shebang-wrapper-test-1.0-r0: task do_install: Failed
ERROR: Task 
(/home/pokybuild/yocto-worker/oe-selftest-fedora/build/build-st-1782300/meta-selftest/recipes-test/wrapper/cmdline-shebang-wrapper-test.bb:do_install)
 failed with exit code '1'



On 30/05/2023 20:01:26-0700, Mark Hatle via lists.openembedded.org wrote:

Changes to eliminate __*.c function usage were based on the patch:

From: JiaLing Zhang 
Subject: [OE-core] [PATCH v4] Fixes pseudo build in loongarch64

Fixes [YOCTO #15110]

Some functions used in the project have been removed from glibc. After the 
removal of these functions,
the architecture in glibc will not include the removed functions.
This patch resolves the usage and compilation issues on the loongarch64 
architecture

Signed-off-by: JiaLing Zhang 

This code is NOT loongarch64 specific, but implements support for newer
glibc where the __*x*stat and __xmknod* functions are no longer present
in headers as of roughly glibc 2.33.

The functions, on x86, x86_64 and aarch64 may still be present for
compatibility but new software should no longer be using it.  Pseudo
can likely change it's default behavior unless support for really old
hosts is still desired.

Signed-off-by: Mark Hatle 
---
  ports/linux/guts/fopen64.c   |  4 +-
  ports/linux/guts/freopen64.c |  4 +-
  ports/linux/guts/fstat.c |  8 ++-
  ports/linux/guts/fstat64.c   | 14 +++-
  ports/linux/guts/fstatat.c   |  9 ++-
  ports/linux/guts/fstatat64.c | 42 +++-
  ports/linux/guts/lstat.c |  2 +-
  ports/linux/guts/lstat64.c   |  2 +-
  ports/linux/guts/mknod.c |  2 +-
  ports/linux/guts/mknodat.c   | 71 +++-
  ports/linux/guts/mkostemp64.c|  2 +-
  ports/linux/guts/openat.c| 18 ++---
  ports/linux/guts/stat.c  |  2 +-
  ports/linux/guts/stat64.c|  2 +-
  ports/linux/old__x/README| 28 
  ports/linux/{ => old__x}/guts/__fxstat.c |  0
  ports/linux/{ => old__x}/guts/__fxstat64.c   |  0
  ports/linux/{ => old__x}/guts/__fxstatat.c   |  0
  ports/linux/{ => old__x}/guts/__fxstatat64.c |  0
  ports/linux/{ => old__x}/guts/__lxstat.c |  0
  ports/linux/{ => old__x}/guts/__lxstat64.c   |  0
  ports/linux/{ => old__x}/guts/__xmknod.c |  0
  ports/linux/{ => old__x}/guts/__xmknodat.c   |  0
  ports/linux/{ => old__x}/guts/__xstat.c  |  0
  ports/linux/{ => old__x}/guts/__xstat64.c|  0
  ports/linux/old__x/guts/fstat.c  | 15 +
  ports/linux/old__x/guts/fstat64.c| 15 +
  ports/linux/old__x/guts/fstatat.c| 15 +
  ports/linux/old__x/guts/fstatat64.c  | 15 +
  ports/linux/old__x/guts/lstat.c  

Re: [OE-core] [PATCH][pseudo] Move __*xstat* and __xmknod functions to new subport 'old__x'

[Mark Hatle via lists.openembedded.org]

On 5/30/23 10:18 PM, Seebs wrote:

On Tue, 30 May 2023 19:54:41 -0700
Mark Hatle  wrote:


-   int existed = (real___xstat64(_STAT_VER, path, ) != -1);
+   int existed = (base_stat64(path, ) != -1);


Honestly, with the benefit of hindsight, I actually can't even think
why I ever thought I should be using those directly instead of through
the base_foo wrappers. Probably for a reason.

... Just in case, definitely don't merge this before testing it in
case there's some insane reason for which the base_* wrappers didn't
work here.


Ya, it's going to require some soak time in the master-next and various builds 
I suspect.

...


diff --git a/ports/linux/old__x/README b/ports/linux/old__x/README
new file mode 100644
index 000..c94413f
--- /dev/null
+++ b/ports/linux/old__x/README
@@ -0,0 +1,28 @@
+Older glibcs contain stat functions such as:
+
+__fxstat
+__fxstatat
+__lxstat
+__xstat
+
+__fxstat64
+__fxstatat64
+__lxstat64
+__xstat64
+
+The format of these functions use the _STAT_VER defintion.  New
glibc no +longer define or utilize these functions, so neither can we.
+
+We only use this subport when the functions are present, this is
checked +by with the existence of _STAT_VER.


"by with the existence"? Looks like a typo.


Ya, I forgot to clean that up, how does this look as a replacement:

Older glibcs contain xstat functions such as:

__fxstat
__fxstatat
__lxstat
__xstat

__fxstat64
__fxstatat64
__lxstat64
__xstat64

These functions are defined when the _STAT_VER is also defined.

Similarly these xmknod functions are also only defined in older glibcs:

__xmknod
__xmknodat

These functions are defind when the _MKNOD_VER is also defined.

This subport is automatically enabled with BOTH _STAT_VER and _MKNOD_VER
are present.  Alternatively if the user force enables subport, it should
work as long as the symbols for these functions are still available.



Anyway, looks reasonable to me, thanks!

-s

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#181986): 
https://lists.openembedded.org/g/openembedded-core/message/181986
Mute This Topic: https://lists.openembedded.org/mt/99234918/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH][pseudo] Move __*xstat* and __xmknod functions to new subport 'old__x'

[Mark Hatle]

This code only switched the usage of the removed __ functions based on 
availability of _STAT_VER and _MKNOD_VER in the ports/linux/subport file.  If 
you change the conditional to 'if true ; then' the system will enable the older 
style code unconditionally.  I did verify that appears to be working for me.


If anyone has a good idea how to adjust the configure script to pass the data to 
the subport file, that would be ideal.  Default behavior of checking the system, 
optional behaving of an --enable-compat-xfunctions -- or something like that.


As for the loongarch64, this code should work there as it's now written.  I do 
not have a way to test this, so I'm hoping JiaLing Zhang will be able to build 
and test this.


All of my testing was accomplished using the 'make test' target.  More extensive 
testing will be needed before we can trust this for OE, but it's probably closer 
to something that does what we need.


(There is still more duplication then I would like.. hopefully reviewers will be 
able to suggest something.)


--Mark

On 5/30/23 10:01 PM, Mark Hatle via lists.openembedded.org wrote:

Changes to eliminate __*.c function usage were based on the patch:

From: JiaLing Zhang 
Subject: [OE-core] [PATCH v4] Fixes pseudo build in loongarch64

Fixes [YOCTO #15110]

Some functions used in the project have been removed from glibc. After the 
removal of these functions,
the architecture in glibc will not include the removed functions.
This patch resolves the usage and compilation issues on the loongarch64 
architecture

Signed-off-by: JiaLing Zhang 

This code is NOT loongarch64 specific, but implements support for newer
glibc where the __*x*stat and __xmknod* functions are no longer present
in headers as of roughly glibc 2.33.

The functions, on x86, x86_64 and aarch64 may still be present for
compatibility but new software should no longer be using it.  Pseudo
can likely change it's default behavior unless support for really old
hosts is still desired.

Signed-off-by: Mark Hatle 
---
  ports/linux/guts/fopen64.c   |  4 +-
  ports/linux/guts/freopen64.c |  4 +-
  ports/linux/guts/fstat.c |  8 ++-
  ports/linux/guts/fstat64.c   | 14 +++-
  ports/linux/guts/fstatat.c   |  9 ++-
  ports/linux/guts/fstatat64.c | 42 +++-
  ports/linux/guts/lstat.c |  2 +-
  ports/linux/guts/lstat64.c   |  2 +-
  ports/linux/guts/mknod.c |  2 +-
  ports/linux/guts/mknodat.c   | 71 +++-
  ports/linux/guts/mkostemp64.c|  2 +-
  ports/linux/guts/openat.c| 18 ++---
  ports/linux/guts/stat.c  |  2 +-
  ports/linux/guts/stat64.c|  2 +-
  ports/linux/old__x/README| 28 
  ports/linux/{ => old__x}/guts/__fxstat.c |  0
  ports/linux/{ => old__x}/guts/__fxstat64.c   |  0
  ports/linux/{ => old__x}/guts/__fxstatat.c   |  0
  ports/linux/{ => old__x}/guts/__fxstatat64.c |  0
  ports/linux/{ => old__x}/guts/__lxstat.c |  0
  ports/linux/{ => old__x}/guts/__lxstat64.c   |  0
  ports/linux/{ => old__x}/guts/__xmknod.c |  0
  ports/linux/{ => old__x}/guts/__xmknodat.c   |  0
  ports/linux/{ => old__x}/guts/__xstat.c  |  0
  ports/linux/{ => old__x}/guts/__xstat64.c|  0
  ports/linux/old__x/guts/fstat.c  | 15 +
  ports/linux/old__x/guts/fstat64.c| 15 +
  ports/linux/old__x/guts/fstatat.c| 15 +
  ports/linux/old__x/guts/fstatat64.c  | 15 +
  ports/linux/old__x/guts/lstat.c  | 15 +
  ports/linux/old__x/guts/lstat64.c| 15 +
  ports/linux/old__x/guts/mknod.c  | 15 +
  ports/linux/old__x/guts/mknodat.c| 15 +
  ports/linux/old__x/guts/mkostemp64.c | 53 +++
  ports/linux/old__x/portdefs.h| 40 +++
  ports/linux/old__x/pseudo_wrappers.c | 48 +
  ports/linux/old__x/wrapfuncs.in  | 18 +
  ports/linux/portdefs.h   | 19 --
  ports/linux/pseudo_wrappers.c| 43 
  ports/linux/subports | 15 +
  ports/linux/wrapfuncs.in | 26 +++
  pseudo_client.h  | 38 +--
  42 files changed, 520 insertions(+), 110 deletions(-)
  create mode 100644 ports/linux/old__x/README
  rename ports/linux/{ => old__x}/guts/__fxstat.c (100%)
  rename ports/linux/{ => old__x}/guts/__fxstat64.c (100%)
  rename ports/linux/{ => old__x}/guts/__fxstatat.c (100%)
  rename ports/linux/{ => old__x}/guts/__fxstatat64.c (100%)
  rename ports/linux/{ => old__x}/guts/__lxstat.c (100%)
  rename ports/linux/{ => old__x}/guts/__lxstat64.c (100%)
  rename ports/linux/{ 

[OE-core] [PATCH][pseudo] Move __*xstat* and __xmknod functions to new subport 'old__x'

[Mark Hatle via lists.openembedded.org]

Changes to eliminate __*.c function usage were based on the patch:

   From: JiaLing Zhang 
   Subject: [OE-core] [PATCH v4] Fixes pseudo build in loongarch64

   Fixes [YOCTO #15110]

   Some functions used in the project have been removed from glibc. After the 
removal of these functions,
   the architecture in glibc will not include the removed functions.
   This patch resolves the usage and compilation issues on the loongarch64 
architecture

   Signed-off-by: JiaLing Zhang 

This code is NOT loongarch64 specific, but implements support for newer
glibc where the __*x*stat and __xmknod* functions are no longer present
in headers as of roughly glibc 2.33.

The functions, on x86, x86_64 and aarch64 may still be present for
compatibility but new software should no longer be using it.  Pseudo
can likely change it's default behavior unless support for really old
hosts is still desired.

Signed-off-by: Mark Hatle 
---
 ports/linux/guts/fopen64.c   |  4 +-
 ports/linux/guts/freopen64.c |  4 +-
 ports/linux/guts/fstat.c |  8 ++-
 ports/linux/guts/fstat64.c   | 14 +++-
 ports/linux/guts/fstatat.c   |  9 ++-
 ports/linux/guts/fstatat64.c | 42 +++-
 ports/linux/guts/lstat.c |  2 +-
 ports/linux/guts/lstat64.c   |  2 +-
 ports/linux/guts/mknod.c |  2 +-
 ports/linux/guts/mknodat.c   | 71 +++-
 ports/linux/guts/mkostemp64.c|  2 +-
 ports/linux/guts/openat.c| 18 ++---
 ports/linux/guts/stat.c  |  2 +-
 ports/linux/guts/stat64.c|  2 +-
 ports/linux/old__x/README| 28 
 ports/linux/{ => old__x}/guts/__fxstat.c |  0
 ports/linux/{ => old__x}/guts/__fxstat64.c   |  0
 ports/linux/{ => old__x}/guts/__fxstatat.c   |  0
 ports/linux/{ => old__x}/guts/__fxstatat64.c |  0
 ports/linux/{ => old__x}/guts/__lxstat.c |  0
 ports/linux/{ => old__x}/guts/__lxstat64.c   |  0
 ports/linux/{ => old__x}/guts/__xmknod.c |  0
 ports/linux/{ => old__x}/guts/__xmknodat.c   |  0
 ports/linux/{ => old__x}/guts/__xstat.c  |  0
 ports/linux/{ => old__x}/guts/__xstat64.c|  0
 ports/linux/old__x/guts/fstat.c  | 15 +
 ports/linux/old__x/guts/fstat64.c| 15 +
 ports/linux/old__x/guts/fstatat.c| 15 +
 ports/linux/old__x/guts/fstatat64.c  | 15 +
 ports/linux/old__x/guts/lstat.c  | 15 +
 ports/linux/old__x/guts/lstat64.c| 15 +
 ports/linux/old__x/guts/mknod.c  | 15 +
 ports/linux/old__x/guts/mknodat.c| 15 +
 ports/linux/old__x/guts/mkostemp64.c | 53 +++
 ports/linux/old__x/portdefs.h| 40 +++
 ports/linux/old__x/pseudo_wrappers.c | 48 +
 ports/linux/old__x/wrapfuncs.in  | 18 +
 ports/linux/portdefs.h   | 19 --
 ports/linux/pseudo_wrappers.c| 43 
 ports/linux/subports | 15 +
 ports/linux/wrapfuncs.in | 26 +++
 pseudo_client.h  | 38 +--
 42 files changed, 520 insertions(+), 110 deletions(-)
 create mode 100644 ports/linux/old__x/README
 rename ports/linux/{ => old__x}/guts/__fxstat.c (100%)
 rename ports/linux/{ => old__x}/guts/__fxstat64.c (100%)
 rename ports/linux/{ => old__x}/guts/__fxstatat.c (100%)
 rename ports/linux/{ => old__x}/guts/__fxstatat64.c (100%)
 rename ports/linux/{ => old__x}/guts/__lxstat.c (100%)
 rename ports/linux/{ => old__x}/guts/__lxstat64.c (100%)
 rename ports/linux/{ => old__x}/guts/__xmknod.c (100%)
 rename ports/linux/{ => old__x}/guts/__xmknodat.c (100%)
 rename ports/linux/{ => old__x}/guts/__xstat.c (100%)
 rename ports/linux/{ => old__x}/guts/__xstat64.c (100%)
 create mode 100644 ports/linux/old__x/guts/fstat.c
 create mode 100644 ports/linux/old__x/guts/fstat64.c
 create mode 100644 ports/linux/old__x/guts/fstatat.c
 create mode 100644 ports/linux/old__x/guts/fstatat64.c
 create mode 100644 ports/linux/old__x/guts/lstat.c
 create mode 100644 ports/linux/old__x/guts/lstat64.c
 create mode 100644 ports/linux/old__x/guts/mknod.c
 create mode 100644 ports/linux/old__x/guts/mknodat.c
 create mode 100644 ports/linux/old__x/guts/mkostemp64.c
 create mode 100644 ports/linux/old__x/portdefs.h
 create mode 100644 ports/linux/old__x/pseudo_wrappers.c
 create mode 100644 ports/linux/old__x/wrapfuncs.in

diff --git a/ports/linux/guts/fopen64.c b/ports/linux/guts/fopen64.c
index e76da69..33ccd3a 100644
--- a/ports/linux/guts/fopen64.c
+++ b/ports/linux/guts/fopen64.c
@@ -11,7 +11,7 @@
struct stat64 buf;
int save_errno;
 
-   int existed = (real___xstat64(_STAT_VER, path, ) != -1);
+   int existe

Re: [OE-core] [PATCH v4] Fixes pseudo build in loongarch64

[Mark Hatle]

I am working on changing the v4 into a more generic implementation based on 
Seebs' and others comments.  I'll likely be sending something to the list as an 
RFC later today or tomorrow.


--Mark

On 5/30/23 11:43 AM, Seebs wrote:

On Tue, 30 May 2023 19:33:03 +0800
zhangjial...@loongson.cn wrote:


+#ifdef __loongarch64


No.

Use the configuration mechanisms or whatever to determine the *actual
distinction you care about*, which is "does glibc have __xstat64", give
that a name, and use the name.

Some day, someone else will make a glibc that doesn't have __xstat64.
The change to adopt that should be one line in configure or the like,
not hundreds of changes scattered throughout the code.


+int existed = (real_stat64(path, ) != -1);
+#else
int existed = (real___xstat64(_STAT_VER, path, ) != -1);
-
+#endif


This looks suspiciously similar to the logic you introduce elsewhere
for base_stat, base_fstat, etcetera.

Reproducing that up here:


+#ifdef __loongarch64
+#define base_fstatat(dirfd, path, buf, flags) real_fstatat64( dirfd,
path, buf, flags)
+#else
  #define base_fstatat(dirfd, path, buf, flags)
real___fxstatat64(_STAT_VER, dirfd, path, buf, flags)
+#endif


Again, shouldn't be using a specific arch flag, should be using a
symbolic flag, but: Given this, maybe we should just be using
base_fstatat, base_stat, etcetera, in these lines, rather than
calling a specific real___foo function directly. Or alternatively,
we should be using pseudo_stat, etcetera, which also exist, and
I honestly don't remember why there are two sets of those.

So the change should look something like

- int existed = ((real___xstat64(_STAT_VER, path, ) != -1);
+ int existed = ((base_stat(path, ) != -1);

with suitable updates to the way we define the base_foo functions,
making them contingent on something like HAVE_XSTAT.

This would cover the vast majority of these changes, but not all of
them.


--- a/ports/linux/guts/fstat.c
+++ b/ports/linux/guts/fstat.c
@@ -7,8 +7,17 @@
   * int fstat(int fd, struct stat *buf)
   *int rc = -1;
   */
-
+#ifdef __loongarch64
+   struct stat64 buf64;
+/* populate buffer with complete data */
+   real_fstat(fd, buf);
+/* obtain fake data */
+   rc = wrap_fstat64(fd, );
+/* overwrite */
+   pseudo_stat32_from64(buf, );
+#else
rc = wrap___fxstat(_STAT_VER, fd, buf);
+#endif
  
  /*	return rc;

   * }


Here, we really do have a meaningful change; in the standard linux port,
we know that we can just forward fstat to __fxstat.

Probably the best model for the canonical way to fix this is to look
at the oldclone/newclone subports. When you have changes this
substantive, we don't want to do them with large inline #ifdefs; we want
to describe these as distinct subports, which have a different set of
functions. So in the hypothetical new "xstat" port, we'd have the
existing fstat and __fxstat wrappers, and in the new "noxstat" port,
we'd just have an fstat wrapper that uses the same logic.


diff --git a/templates/wrapfuncs.c b/templates/wrapfuncs.c
index 93bb671..14a42e2 100644
--- a/templates/wrapfuncs.c
+++ b/templates/wrapfuncs.c
@@ -13,7 +13,9 @@
   * script if you want to modify this. */
  @body
  
+#ifndef __loongarch64

  static ${type} (*real_${name})(${decl_args}) = ${real_init};
+#endif
  
  ${maybe_skip}


I don't understand how this part can work at all; I was under the
impression that we actually did in fact use these real_foo objects,
and this seems to remove them all, so I don't even get how this
is compiling or running, because after this, there's no real_foo
function pointers at all?

Unfortunately, it turns out bitrot has taken my laptop and I can't
build pseudo on it natively because _STAT_VER no longer exists. So
I think the hypothetical distant future in which we conceivably
care about a system on which the real___xstatat(_STAT_VER, ...) calls
fail and isn't __longarch64 actually happened over a year ago and
we just didn't notice.

"Oops."

-s






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#181957): 
https://lists.openembedded.org/g/openembedded-core/message/181957
Mute This Topic: https://lists.openembedded.org/mt/99217464/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] Initramfs recipes - How to set IMAGE_FSTYPES

[Mark Hatle]

A little background, looking at oe-core and poky for background, I see the 
initramfs images contain:


IMAGE_FSTYPES = "${INITRAMFS_FSTYPES}"

I see this in both oe-core 
meta/recipes-core/images/core-image-minimal-initramfs.bb and 
core-image-tiny-initramfs.bb.


The "meta-security" layer also seems to the do the same in it's 
'dm-verify-image-iniramfs.bb' recipe as well.



However, when I look at meta-openembedded/meta-initramfs (initramfs-debug-image 
and initramfs-kexecboot-image), I see:


# Some BSPs use IMAGE_FSTYPES_ which would override
# an assignment to IMAGE_FSTYPES so we need anon python
python() {
   d.setVar("IMAGE_FSTYPES", d.getVar("INITRAMFS_FSTYPES"))
}


This is the exact issue I'm trying to understand.  (Ignoring the comment above 
uses the older style override syntax), the issue I have is someone is setting:


IMAGE_FSTYPES: = "something"
INITRAMFS_FSTYPES: = "something else"

Then the initramfs recipes in oe-core are failing to set IMAGE_FSTYPES to 
INITRAMFS_FSTYPES due to the IMAGE_FSTYPES override, which causes the types to 
not be correct.



My question then is, is setting IMAGE_FSTYPES: valid, if not should be 
add a sanity check to tell people to NOT do it?  If it is valid to do, is the 
workaround/fix in meta-openembedded the right solution?   If using the python 
approach is correct, then I'm happy to generate patches to oe-core but I'm just 
not sure what the expected usage is.


Thanks!
--Mark

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#181028): 
https://lists.openembedded.org/g/openembedded-core/message/181028
Mute This Topic: https://lists.openembedded.org/mt/98769851/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [master][mickledore][PATCH v2] sanity.bbclass: Update minimum gcc version to 8.0

[Mark Hatle]

From: Mark Hatle 

With a gcc older then 8.0, mesa-native will fail to build with the error:
  sorry, unimplemented: non-trivial designated initializers not supported

According to https://docs.mesa3d.org/install.html?highlight=gcc+version#compile
the required minimum compiler version is now GCC 8.0.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
v2:
 Fixed typo in commit message.  meta -> mesa

 meta/classes-global/sanity.bbclass | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/meta/classes-global/sanity.bbclass 
b/meta/classes-global/sanity.bbclass
index 2d1ff7072c..abb52fbe21 100644
--- a/meta/classes-global/sanity.bbclass
+++ b/meta/classes-global/sanity.bbclass
@@ -475,7 +475,7 @@ def check_wsl(d):
 bb.warn("You are running bitbake under WSLv2, this works properly 
but you should optimize your VHDX file eventually to avoid running out of 
storage space")
 return None
 
-# Require at least gcc version 7.5.
+# Require at least gcc version 8.0
 #
 # This can be fixed on CentOS-7 with devtoolset-6+
 # https://www.softwarecollections.org/en/scls/rhscl/devtoolset-6/
@@ -488,8 +488,8 @@ def check_gcc_version(sanity_data):
 
 build_cc, version = oe.utils.get_host_compiler_version(sanity_data)
 if build_cc.strip() == "gcc":
-if bb.utils.vercmp_string_op(version, "7.5", "<"):
-return "Your version of gcc is older than 7.5 and will break 
builds. Please install a newer version of gcc (you could use the project's 
buildtools-extended-tarball or use scripts/install-buildtools).\n"
+if bb.utils.vercmp_string_op(version, "8.0", "<"):
+return "Your version of gcc is older than 8.0 and will break 
builds. Please install a newer version of gcc (you could use the project's 
buildtools-extended-tarball or use scripts/install-buildtools).\n"
 return None
 
 # Tar version 1.24 and onwards handle overwriting symlinks correctly
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#179695): 
https://lists.openembedded.org/g/openembedded-core/message/179695
Mute This Topic: https://lists.openembedded.org/mt/98061607/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [master][mickledore][PATCH] sanity.bbclass: Update minimum gcc version to 8.0

[Mark Hatle]

From: Mark Hatle 

With a gcc older then 8.0, meta-native will fail to build with the error:
  sorry, unimplemented: non-trivial designated initializers not supported

According to https://docs.mesa3d.org/install.html?highlight=gcc+version#compile
the required minimum compiler version is now GCC 8.0.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/classes-global/sanity.bbclass | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/meta/classes-global/sanity.bbclass 
b/meta/classes-global/sanity.bbclass
index 2d1ff7072c..abb52fbe21 100644
--- a/meta/classes-global/sanity.bbclass
+++ b/meta/classes-global/sanity.bbclass
@@ -475,7 +475,7 @@ def check_wsl(d):
 bb.warn("You are running bitbake under WSLv2, this works properly 
but you should optimize your VHDX file eventually to avoid running out of 
storage space")
 return None
 
-# Require at least gcc version 7.5.
+# Require at least gcc version 8.0
 #
 # This can be fixed on CentOS-7 with devtoolset-6+
 # https://www.softwarecollections.org/en/scls/rhscl/devtoolset-6/
@@ -488,8 +488,8 @@ def check_gcc_version(sanity_data):
 
 build_cc, version = oe.utils.get_host_compiler_version(sanity_data)
 if build_cc.strip() == "gcc":
-if bb.utils.vercmp_string_op(version, "7.5", "<"):
-return "Your version of gcc is older than 7.5 and will break 
builds. Please install a newer version of gcc (you could use the project's 
buildtools-extended-tarball or use scripts/install-buildtools).\n"
+if bb.utils.vercmp_string_op(version, "8.0", "<"):
+return "Your version of gcc is older than 8.0 and will break 
builds. Please install a newer version of gcc (you could use the project's 
buildtools-extended-tarball or use scripts/install-buildtools).\n"
 return None
 
 # Tar version 1.24 and onwards handle overwriting symlinks correctly
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#179672): 
https://lists.openembedded.org/g/openembedded-core/message/179672
Mute This Topic: https://lists.openembedded.org/mt/98052854/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] mesa-native fails on master

[Mark Hatle]

I'm fine with saying either Ubuntu 18.04 is no longer 'supported', or running 
some kind of check that says gcc 7.5.0 is too old..  But I'm not sure what 
minimum (host) compiler version would be needed to avoid this error.


I'm happy to work up a patch to adjust the minimum gcc version, if someone can 
give me a bit of a clue what the appropriate minimum should be.


The current check (in sanity.bbclass) looks for 7.5 or newer.

--Mark

On 4/3/23 9:48 PM, Khem Raj wrote:

On Mon, Apr 3, 2023 at 7:39 PM Chen Qi  wrote:


I just met the same issue. My host is ubuntu18, gcc version is 7.5.0.

Regards,
Qi

-Original Message-
From: openembedded-core@lists.openembedded.org 
 On Behalf Of Mark Hatle
Sent: Tuesday, April 4, 2023 10:25 AM
To: Patches and discussions about the oe-core layer 

Subject: [OE-core] mesa-native fails on master

It's been a few weeks since I last built master, but starting with today's pull 
I'm getting errors building mesa-native -- which pretty much kills the whole 
build.

I'm on an Ubuntu 18.04 host, and have the 'buildtools-tarball' from the last
4.1.3 loaded so I can get to the minimum version of python to run a build.

(I tried with buildtools-tarball-extended and that DOES appear to work.  If 
this is expected, you can ignore the rest of this.  But I expected the system 
to complain before trying to build everything if the compiler wasn't new 
enough.)


The error is:

brw_simd_selection.cpp:205:7: sorry, unimplemented: non-trivial designated 
initializers not supported


The host gcc is 7.5.0

What is strange is I had 2 successful builds, and then it started failing.  All
fresh builds.  I've no idea why one was successful and the others failed.  I'm
wondering if maybe there is some sort of race in mesa, but lowered my parallel
build (-j) to 1 and I'm still getting it.



You need a newer gcc compiler > 7.x
I just abandoned using ubuntu 18.04
see - 
https://lists.openembedded.org/g/openembedded-core/message/178783?p=%2C%2C%2C20%2C0%2C0%2C0%3A%3Acreated%2C0%2Cnon-trivial+designated+initializers+not+supported%2C20%2C2%2C0%2C97647023



Reproducer:

git clone https://git.yoctoproject.org/poky

cd poky

wget
http://downloads.yoctoproject.org/releases/yocto/yocto-4.1.3/buildtools/x86_64-buildtools-nativesdk-standalone-4.1.3.sh

bash x86_64-buildtools-nativesdk-standalone-4.1.3.sh -d buildtools -y

. ./buildtools/environment-...

. ./oe-init-build-env

bitbake mesa-native










-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#179670): 
https://lists.openembedded.org/g/openembedded-core/message/179670
Mute This Topic: https://lists.openembedded.org/mt/98052337/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] mesa-native fails on master

[Mark Hatle]

It's been a few weeks since I last built master, but starting with today's pull 
I'm getting errors building mesa-native -- which pretty much kills the whole build.


I'm on an Ubuntu 18.04 host, and have the 'buildtools-tarball' from the last 
4.1.3 loaded so I can get to the minimum version of python to run a build.


(I tried with buildtools-tarball-extended and that DOES appear to work.  If this 
is expected, you can ignore the rest of this.  But I expected the system to 
complain before trying to build everything if the compiler wasn't new enough.)



The error is:

brw_simd_selection.cpp:205:7: sorry, unimplemented: non-trivial designated 
initializers not supported



The host gcc is 7.5.0

What is strange is I had 2 successful builds, and then it started failing.  All 
fresh builds.  I've no idea why one was successful and the others failed.  I'm 
wondering if maybe there is some sort of race in mesa, but lowered my parallel 
build (-j) to 1 and I'm still getting it.



Reproducer:

git clone https://git.yoctoproject.org/poky

cd poky

wget 
http://downloads.yoctoproject.org/releases/yocto/yocto-4.1.3/buildtools/x86_64-buildtools-nativesdk-standalone-4.1.3.sh


bash x86_64-buildtools-nativesdk-standalone-4.1.3.sh -d buildtools -y

. ./buildtools/environment-...

. ./oe-init-build-env

bitbake mesa-native



-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#179657): 
https://lists.openembedded.org/g/openembedded-core/message/179657
Mute This Topic: https://lists.openembedded.org/mt/98052337/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH] tcf-agent: Update to current version

[Mark Hatle]

From: Mark Hatle 

While there has not been an official release in some time, the latest version
fixes a large number of bugs and adds support for Dwarf 5, among other
things.  (Dwarf 5 is default in gcc 12 and newer.)

Short change log since the last version:
TCF Agent: Narrow down LoadLibrary search scope
TCF Agent: faster breakpoint planting
Bug 581214 - Expression evaluation fails for a data member that is defined 
in a structure, union or class that is of type DW_AT_data_bit_offset attribute 
(DWARF5)
TCF Agent: reduced memory footprint
TCF Agent: fixed build error on CygWin: CYGWIN_VERSION_CYGWIN_CONV undefined
TCF Agent: workaround for GCC compilation error: function may return 
address of local variable
Bug 581034 - "long long int" data type can't be recognized from dwarf5 
debug info generated by llvm15
TCF Agent: Removal of unused variable causing clang-15 build error
TCF Agent: fixed: malformed HTTP request can crash the agent
TCF Agent: fixed assertion failure in the Breakpoints service
TCF Agent: GDB Remote Serial Protocol: fixed handling of process ID 0
Bug 580644 - steps into the function of shared library does not work on PPC
TCF Agent: add PowerPC 32-bit and big-endian builds in the Dockerfile
TCF Agent: check glibc version when calling pthread_setname_np()
Bug 580600 - -Wmisleading-indentation issue for gcc version less than 6.0
Bug 580489 - failed to handle .debug_rnglists section sometimes
Bug 580450 - failed to read the full compilation unit tag in dwarf 5 when 
customized sections
Bug 580414 - failed to handle debug sections for DWARF 5
Bug 580326 - Can not display the type of global variable defined in a 
sharedlib
Fixed text formatting in Makefile.inc
Bug 580279 - VERSION grep in agent/Makefile.inc need update for vxWorks
Bug 580089 - pid2id() failed to get a right id
TCF Agent: improved comments
TCF Agent: a few more asserts
TCF Agent: improved X86 disassembler
Bug 580002 - [tcf-dev] Speedup compute_reverse_lookup_indices
Bug 579989 - compile error by vs2008 on windows
TCF Agent: improved X86 disassembler
Bug 579947 - NULL pointer caused the tcf-server crashed when load ELF file 
that compiled by LLVM 14 with dwarf-5
TCF Agent: fixed Coverity warning
TCF Agent: fixed Coverity warning
TCF Agent: new function in HTTP server API: closed() call-back
Merge "TCF Agent: Fix potential memory/resource leaks"
TCF Agent: Fix potential memory/resource leaks
TCF Agent: Fix unsafe strcpy
TCF Agent: Respect certain ENABLE_* macros
Bug 579412 - incorrect or missing copyright information
TCF Agent: Lazy PC initialization
Bug 579362 - the process IDs should be pid_t instead of UINT32 in waitpid.c 
and waitpid.h files
Bug 579378 - update copyright year to 2022
Bug 579274 - unknown option -Wmisleading-indentation issue for gcc version 
less than 6.0
TCF Agent: ARM v8 stack crawl: fixed handling of ERET instruction
TCF Agent: fixed handling of line info file names when file contains a mix 
DWARF 5 and DWARF 3
TCF Tests: fixed incorrect error reports in DWARF reader test
TCF Agent: disabled bogus warnings misleading-indentation
TCF Agent: fixed handling of line info file names in DWARF 5
TCF Agent: Expressions: fixed handling of a static field of a struct stored 
in a register
TCF Agent: fixed error when building with c++-11
TCF Agent: DWARF 5: improved handling of location expressions
TCF Agent: DWARF reader: fixed regression - "FORM_FLAG expected" error
TCF Agent: DWARF reader: new object flag DOIF_inlined
TCF Tests: improved tests of var declarations
TCF Agent: fixed regression: memory corruption in the symbols proxy
TCF Agent: fixed issues with handling GCC-11 debug info
TCF Agent: fixed warning: ISO C90 forbids mixed declarations and code
TCF Agent: Fix stack frame cleanup on error
TCF Agent: Add missing parentheses
Bug 578201 - Sometimes variables show as "N/A - Cannot read target memory. 
Input/output error"
Bug 577936 - dprintf not work as expected
TCF Agent: lazy initialization in rand32()
TCF Agent: new API function: rand32()
TCF Agent: fixed possible buffer overflow when calling fscanf()
Bug 577174 - Sometimes variables show as "N/A - Value of register is 
unknown in the selected frame"
Bug 577064 - Union type variables don’t show correctly if based on a 
register
TCF Agent: fixed: tmp_vprintf and loc_vprintf can segfault on Linux
Bug 577001 - DW_AT_high_pc with DW_FORM_udata form not handled
TCF Agent: fixed pthread_cond_timedwait() on Windows
TCF Agent: Expressions service: improved error message

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 .../tcf-agent/tcf-agent/ldflags.patch | 29 ++-
 .../tcf-agent/tcf-agent_git.bb

Re: [OE-core] [RFC PATCH v2] qemu: update 7.1.0 -> 7.2.0

[Mark Hatle]

On 12/16/22 9:45 AM, Alexander Kanavin wrote:

Disable slirp by default: qemu no longer carries libslirp in-tree,
and enabling slirp requires providing external libslirp first
(available from e.g. meta-virtualization).


I think disabling slirp (by default) is going to be problematic.  Most of my 
configurations and those of people I work with use slirp.  Forcing people to 
include meta-virtualization in all configurations may not be reasonable either.


I suggest that we bring libslirp from meta-virtualization to oe-core to preserve 
existing behavior.


--Mark


Another noteworthy change is:

x86: TCG support for AVX, AVX2, F16C, FMA3 and VAES instructions

... which means both meta-intel and qemu x86 targets can
now fully utilize Haswell-and-later instruction set with benefits
for performance in emulation and on silicon.

Changelog:
https://wiki.qemu.org/ChangeLog/7.2


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#174763): 
https://lists.openembedded.org/g/openembedded-core/message/174763
Mute This Topic: https://lists.openembedded.org/mt/95712037/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-Core][master][PATCH] openssh: remove RRECOMMENDS to rng-tools for sshd package

[Mark Hatle]

In the kernel is the jitter entropy system.  It should work on all platforms 
that have high res timers available.  (This is the same mechanism that haveged 
was using before as well.  So no change in the RNG quality, just now built into 
the kernel itself.)


The only place we've observed an issue with the new approach is on a qemu 
machine that was moderately loaned.  The jitter was apparently too large (our 
theory) and caused the crng init to happen VERY slowly.  (This would not have 
been resolved using rngd.)


All-in-all, I thought this change had already been made, and think it should go 
in.

(We tested this on microblaze, and performance with and without haveged was the 
same.  I don't have number, but I expect the same behaviors on any architecture.)


--Mark

On 12/9/22 11:02 AM, Khem Raj wrote:

would be good to know some numbers on non-arm/non-x86 systems too.

On Thu, Dec 8, 2022 at 11:05 PM Xiangyu Chen
 wrote:


It appears that rngd is not needed as of linux-5.6 and later[1]
and should not be installed by default since the purpose of rngd
is to provide additional trusted sources of entropy.

We did some testing on real hardware, the result seems to support that
we no longer need rngd by default on kernel v5.6 and later.

Testing result as below:

1. observing the crng init stage.
  the "random: crng init done" always available before fs being mounted.

2. generating random number without rngd.
  testing command: dd if=/dev/random of=/dev/null status=progress
on Marvell CN96xx RDB board, speed almost 20.4 MB/s without block
on NXP i.mx6q board, speed almost 31.9 MB/s without block
on qemu x86-64, speed almost 2.6MB/s without block

3. using rngtest command without rngd
  testing command: rngtest -c 1000 https://github.com/torvalds/linux/commit/30c08efec8884fb106b8e57094baa51bb4c44e32

Signed-off-by: Xiangyu Chen 
---
  meta/recipes-connectivity/openssh/openssh_9.1p1.bb | 9 +
  1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/meta/recipes-connectivity/openssh/openssh_9.1p1.bb 
b/meta/recipes-connectivity/openssh/openssh_9.1p1.bb
index 85f97b1bbb..23ae8d5b0c 100644
--- a/meta/recipes-connectivity/openssh/openssh_9.1p1.bb
+++ b/meta/recipes-connectivity/openssh/openssh_9.1p1.bb
@@ -52,15 +52,12 @@ SYSTEMD_SERVICE:${PN}-sshd = "sshd.socket"

  inherit autotools-brokensep ptest

-PACKAGECONFIG ??= "rng-tools"
+PACKAGECONFIG ??= ""
  PACKAGECONFIG[kerberos] = "--with-kerberos5,--without-kerberos5,krb5"
  PACKAGECONFIG[ldns] = "--with-ldns,--without-ldns,ldns"
  PACKAGECONFIG[libedit] = "--with-libedit,--without-libedit,libedit"
  PACKAGECONFIG[manpages] = "--with-mantype=man,--with-mantype=cat"

-# Add RRECOMMENDS to rng-tools for sshd package
-PACKAGECONFIG[rng-tools] = ""
-
  EXTRA_AUTORECONF += "--exclude=aclocal"

  # login path is hardcoded in sshd
@@ -160,10 +157,6 @@ FILES:${PN}-keygen = "${bindir}/ssh-keygen"

  RDEPENDS:${PN} += "${PN}-scp ${PN}-ssh ${PN}-sshd ${PN}-keygen 
${PN}-sftp-server"
  RDEPENDS:${PN}-sshd += "${PN}-keygen ${@bb.utils.contains('DISTRO_FEATURES', 
'pam', 'pam-plugin-keyinit pam-plugin-loginuid', '', d)}"
-RRECOMMENDS:${PN}-sshd:append:class-target = "\
-${@bb.utils.filter('PACKAGECONFIG', 'rng-tools', d)} \
-"
-
  # gdb would make attach-ptrace test pass rather than skip but not worth the 
build dependencies
  RDEPENDS:${PN}-ptest += "${PN}-sftp ${PN}-misc ${PN}-sftp-server make sed sudo 
coreutils"

--
2.34.1









-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#174478): 
https://lists.openembedded.org/g/openembedded-core/message/174478
Mute This Topic: https://lists.openembedded.org/mt/95556189/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [master,langdale][PATCH] qemurunner.py: Support fallback to older QEMU qmp library

[Mark Hatle]

Ping, any comments on this?

On 11/21/22 1:20 PM, Mark Hatle wrote:

From: Mark Hatle 

Some vendor versions of QEMU may be on older versions that do not have
qmp.legacy, but instead of qmp.  Default to the integrated library
versions, fall back to the older one and then if neither is available
catch the exception and provide a more human readable error.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
  meta/lib/oeqa/utils/qemurunner.py | 8 +++-
  1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/utils/qemurunner.py 
b/meta/lib/oeqa/utils/qemurunner.py
index f175f8a1de..436caef273 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -328,8 +328,14 @@ class QemuRunner:
  try:
  os.chdir(os.path.dirname(qmp_port))
  try:
-   from qmp.legacy import QEMUMonitorProtocol
+   try:
+   from qmp.legacy import QEMUMonitorProtocol
+   except ModuleNotFoundError:
+   from qmp import QEMUMonitorProtocol
 self.qmp = QEMUMonitorProtocol(os.path.basename(qmp_port))
+except ModuleNotFoundError as msg:
+self.logger.warning("Failed to load QEMUMonitorProtocol from 
qmp.legacy or qmp: %s" % (msg))
+return False
  except OSError as msg:
  self.logger.warning("Failed to initialize qemu monitor socket: %s 
File: %s" % (msg, msg.filename))
  return False






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#174246): 
https://lists.openembedded.org/g/openembedded-core/message/174246
Mute This Topic: https://lists.openembedded.org/mt/95180018/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [master,langdale][PATCH] qemurunner.py: Support fallback to older QEMU qmp library

[Mark Hatle]

From: Mark Hatle 

Some vendor versions of QEMU may be on older versions that do not have
qmp.legacy, but instead of qmp.  Default to the integrated library
versions, fall back to the older one and then if neither is available
catch the exception and provide a more human readable error.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/lib/oeqa/utils/qemurunner.py | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/meta/lib/oeqa/utils/qemurunner.py 
b/meta/lib/oeqa/utils/qemurunner.py
index f175f8a1de..436caef273 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -328,8 +328,14 @@ class QemuRunner:
 try:
 os.chdir(os.path.dirname(qmp_port))
 try:
-   from qmp.legacy import QEMUMonitorProtocol
+   try:
+   from qmp.legacy import QEMUMonitorProtocol
+   except ModuleNotFoundError:
+   from qmp import QEMUMonitorProtocol
self.qmp = QEMUMonitorProtocol(os.path.basename(qmp_port))
+except ModuleNotFoundError as msg:
+self.logger.warning("Failed to load QEMUMonitorProtocol from 
qmp.legacy or qmp: %s" % (msg))
+return False
 except OSError as msg:
 self.logger.warning("Failed to initialize qemu monitor socket: 
%s File: %s" % (msg, msg.filename))
 return False
-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#173661): 
https://lists.openembedded.org/g/openembedded-core/message/173661
Mute This Topic: https://lists.openembedded.org/mt/95180018/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] Running weston as non-weston user

[Mark Hatle]

On 11/1/22 3:03 PM, Gundlupet Raju, Sandeep wrote:

Hello,

We wanted to know what the actual use case is for running weston as
weston user only in OE core
http://cgit.openembedded.org/openembedded-core/tree/meta/recipes-graphics/wayland/weston-init/weston.service#n44


How is this expected to be used for another user.  Is this intended to
be run as the primary machine user or is there another approach? For a
system where weston starts itself, it’s expected to start with the
weston user only?

Can we add a existing user to weston group and make it to work?


An additional question:

For a system w/o mouse and keyboard, can we ssh in and execute our wayland tests 
from remote?  If so, what user should we be using (weston), can we define our 
own user?


--Mark


Thanks,
Sandeep

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#172385): 
https://lists.openembedded.org/g/openembedded-core/message/172385
Mute This Topic: https://lists.openembedded.org/mt/94719795/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH v2] mesa: Add native patch via a variable

[Mark Hatle]

On 10/28/22 5:59 AM, Alexander Kanavin wrote:

Thanks. What I would want to understand is whether this arrangement
should have its own
MACHINE_FEATURE, e.g. 'vendor-egl' or similar, and then mesa proper
would be configured accordingly to
disable bits that come from the vendor.

Is there a particular need for a separate recipe?


When libmali support is enabled in our configuration we do the following:

define a mali400 MACHINEOVERRIDE

Then we use PREFERRED_PROVIDER to switch:

https://github.com/Xilinx/meta-xilinx/blob/master/meta-xilinx-core/conf/machine/include/machine-xilinx-default.inc#L25

virtual/libgles1
virtual/libgles2
virtual/egl
virtual/libgl
virtual/mesa

So the mesa-gl

the PROVIDES are:

virtual/libgl virtual/mesa

while in mesa:

PROVIDES = " \
${@bb.utils.contains('PACKAGECONFIG', 'opengl', 'virtual/libgl', '', d)} \
${@bb.utils.contains('PACKAGECONFIG', 'gles', 'virtual/libgles1 
virtual/libgles2 virtual/libgles3', '', d)} \

${@bb.utils.contains('PACKAGECONFIG', 'egl', 'virtual/egl', '', d)} \
${@bb.utils.contains('PACKAGECONFIG', 'gbm', 'virtual/libgbm', '', d)} \
virtual/mesa \
"

While it might be possible to switch around the PROVIDES using distibution 
configuration or similar, this has been difficult to get right in the past. 
This is why the split happened, it's one or the other based on the configuration.


--Mark


Alex

On Thu, 27 Oct 2022 at 02:46, Mark Hatle  wrote:




On 10/26/22 1:01 PM, Alexander Kanavin wrote:

Is there a combination of libmali recipe and mesa-gl somewhere public
so we could try to pick it apart and see what links to where?


libmali is ALWAYS implementation specific.  You will need to pick a vendor and
build for it.  What is generic is the application interface (from what I've been
told.). I can give you the AMD (Xilinx) instructions.  But you likely won't be
able to actually execute the code without a corresponding board.  (AFAIK QEMU
doesn't have any mali400 emulation.)

And to be clear, libmali provides the interfaces, but it is NOT a replacement to
mesa, it works WITH mesa, just the 'gl' part (not the gles part).

The following is an abbreviated attempt to show how the pieces interact with
each other.  It may not be 100% correct, I am definitely not an expert at this.
(For example, I don't understand why it's "DRM" in some places but "DRI" in
others.  These seem to be 'generally' the same systems.)  But it's how the parts
and pieces appear to be put together based on a functioning system.


In a 'normal' system (mesa):

mesa provides:
libglapi.so.0
libEGL.so.1
libGLESv2.so.2
(and others)

The CONFIG_DRM_LIMA backend and whatever the display port DRM (CONFIG_DRM_XLNX)
go through a dlopened backend module, which USUALLY talks to a kernel module for
any hardware access through a standard (known) kernel interface.

In our case the dynamic modules would be the lima_dri.so and xlnx_dri.so.

glmark2-es2-wayland ->
libwayland-client
libwayland-cursor
libwayland-egl.so.1

I believe the libwayland-egl talks to weston "somehow" for the egl interfaces,
via the libwayland-client?  but I'm fuzzy here.

Weston has two related items:

drm-backend.o ->
libglapi.so.0 (dlopened)

This bit uses mesa to access the DRM backend to actually display the stuff..

gl-backend.so ->
libwayland-server
libEGL.so.1
libGLESv2.so.2

This backend is used to render the items.


So the glmark2 does it's benchmarking and renders (gl-backend) through Weston
backends to the display port (drm-backend).

In the above case it all comes from mesa, the drm and libELG/libGLESv2 parts.



The following shows a libmali configured system as a contrasts to the above:

libmali-xlnx provides:

libEGL.so.1
libGLESv1_CM.so.1
libGLESv2.so.2
libmali.so.9
libgbm.so.1

(also provides specific backends for fbdev, headless, wayland and x11)


It talks to a backend kernel module called 'mali.ko'

mesa provides:

libGL.so.1
libglapi.so.0

and also swrast dri interface

glmark2-es2-wayland ->
libwayland-client
libwayland-cursor
libwayland-egl.so.1

(same as above)

drm-backend.o ->
libwayland-server
libdrm.so.2
libMali.so.9

gl-backend.so ->
libwayland-server
libMali.so.9

So in the above, libMali is providing all of the APIs of libEGL and libGLESv2,
as well as (enough of) libglapi for the built in raster implementation.


Compiling things like Chromium that will link directly, you move from linking to
libGL.so from mesa to linking in libMali, along with libGL from mesa.  The
libMali will provide the core set of APIs, while the libGL will augement them
with additional APIs.

Cairo is another utility that similar modifies in this way.


To re-iterate, it's NOT a "swap libmali" and re-use everything.  You need to
rebuild anything that links to libgles or libgl or even libglapi with this new
configuration.  It's API compatible, NOT ABI compa

Re: [OE-core] [PATCH v2] mesa: Add native patch via a variable

[Mark Hatle]

On 10/26/22 1:01 PM, Alexander Kanavin wrote:

Is there a combination of libmali recipe and mesa-gl somewhere public
so we could try to pick it apart and see what links to where?


libmali is ALWAYS implementation specific.  You will need to pick a vendor and 
build for it.  What is generic is the application interface (from what I've been 
told.). I can give you the AMD (Xilinx) instructions.  But you likely won't be 
able to actually execute the code without a corresponding board.  (AFAIK QEMU 
doesn't have any mali400 emulation.)


And to be clear, libmali provides the interfaces, but it is NOT a replacement to 
mesa, it works WITH mesa, just the 'gl' part (not the gles part).


The following is an abbreviated attempt to show how the pieces interact with 
each other.  It may not be 100% correct, I am definitely not an expert at this. 
(For example, I don't understand why it's "DRM" in some places but "DRI" in 
others.  These seem to be 'generally' the same systems.)  But it's how the parts 
and pieces appear to be put together based on a functioning system.



In a 'normal' system (mesa):

mesa provides:
  libglapi.so.0
  libEGL.so.1
  libGLESv2.so.2
  (and others)

The CONFIG_DRM_LIMA backend and whatever the display port DRM (CONFIG_DRM_XLNX) 
go through a dlopened backend module, which USUALLY talks to a kernel module for 
any hardware access through a standard (known) kernel interface.


In our case the dynamic modules would be the lima_dri.so and xlnx_dri.so.

glmark2-es2-wayland ->
  libwayland-client
  libwayland-cursor
  libwayland-egl.so.1

I believe the libwayland-egl talks to weston "somehow" for the egl interfaces, 
via the libwayland-client?  but I'm fuzzy here.


Weston has two related items:

drm-backend.o ->
  libglapi.so.0 (dlopened)

This bit uses mesa to access the DRM backend to actually display the stuff..

gl-backend.so ->
  libwayland-server
  libEGL.so.1
  libGLESv2.so.2

This backend is used to render the items.


So the glmark2 does it's benchmarking and renders (gl-backend) through Weston 
backends to the display port (drm-backend).


In the above case it all comes from mesa, the drm and libELG/libGLESv2 parts.



The following shows a libmali configured system as a contrasts to the above:

libmali-xlnx provides:

libEGL.so.1
libGLESv1_CM.so.1
libGLESv2.so.2
libmali.so.9
libgbm.so.1

(also provides specific backends for fbdev, headless, wayland and x11)


It talks to a backend kernel module called 'mali.ko'

mesa provides:

libGL.so.1
libglapi.so.0

and also swrast dri interface

glmark2-es2-wayland ->
  libwayland-client
  libwayland-cursor
  libwayland-egl.so.1

(same as above)

drm-backend.o ->
  libwayland-server
  libdrm.so.2
  libMali.so.9

gl-backend.so ->
  libwayland-server
  libMali.so.9

So in the above, libMali is providing all of the APIs of libEGL and libGLESv2, 
as well as (enough of) libglapi for the built in raster implementation.



Compiling things like Chromium that will link directly, you move from linking to 
libGL.so from mesa to linking in libMali, along with libGL from mesa.  The 
libMali will provide the core set of APIs, while the libGL will augement them 
with additional APIs.


Cairo is another utility that similar modifies in this way.


To re-iterate, it's NOT a "swap libmali" and re-use everything.  You need to 
rebuild anything that links to libgles or libgl or even libglapi with this new 
configuration.  It's API compatible, NOT ABI compatible!



I hope this helps.  As far as letting you play with it, I'm working through the 
final stuff and should have something I can push to langdale/master on the 
meta-xilinx and meta-xilinx-tools soon.


--Mark


My (perhaps confused) understanding is similar to Joshua's: 'gl' is
basically 'opengl in x11' or 'opengl rendered to memory', and if a
system uses neither, then it is not needed.

Alex

On Wed, 26 Oct 2022 at 19:48, Joshua Watt  wrote:


On Wed, Oct 26, 2022 at 12:21 PM Mark Hatle
 wrote:




On 10/26/22 11:03 AM, Alexander Kanavin wrote:

On Wed, 26 Oct 2022 at 16:35, Mark Hatle  wrote:

(I just saw this, so a little late on the reply, but..)

mesa-gl is ABSOLUTELY still being used.  It's needed for libmali usage.  Not
everyone wants to use lima support for graphics.

It was broken into two separate packages so that is was VERY clear if you were
using mesa for 'everything' (mesa), or mesa JUST for 'gl' (not gles).


Okay, I'm not going to propose this, but can you clarify, what is the
current use for 'gl'? Is it just opengl 3d in standalone x server
based systems (which is slowly dying), or is there something else to
it?


Are you specifically talking about libmali with the following statements?

I would say that in general libmali is probably not doing things in
the "normal" way if so, although I'm not trying to say that it isn't a
legitimate way to do it.



Anything that needs "OpenGL", i.e. Wayland/Weston, X1

Re: [OE-core] [PATCH v2] mesa: Add native patch via a variable

[Mark Hatle]

On 10/26/22 12:48 PM, Joshua Watt wrote:

On Wed, Oct 26, 2022 at 12:21 PM Mark Hatle
 wrote:




On 10/26/22 11:03 AM, Alexander Kanavin wrote:

On Wed, 26 Oct 2022 at 16:35, Mark Hatle  wrote:

(I just saw this, so a little late on the reply, but..)

mesa-gl is ABSOLUTELY still being used.  It's needed for libmali usage.  Not
everyone wants to use lima support for graphics.

It was broken into two separate packages so that is was VERY clear if you were
using mesa for 'everything' (mesa), or mesa JUST for 'gl' (not gles).


Okay, I'm not going to propose this, but can you clarify, what is the
current use for 'gl'? Is it just opengl 3d in standalone x server
based systems (which is slowly dying), or is there something else to
it?


Are you specifically talking about libmali with the following statements?

I would say that in general libmali is probably not doing things in
the "normal" way if so, although I'm not trying to say that it isn't a
legitimate way to do it.


https://github.com/Xilinx/meta-xilinx/blob/master/meta-xilinx-core/recipes-graphics/libgles/libmali-xlnx.bb

libmali is providing all of the ELG/GLES/GLES2 and KHR headers.  It is providing 
the libegl, libgles1, libgles2, libgbm.  Also provides interfaces to fbdev, X11, 
and Wayland usage.


But all of this is specific to ONLY gles 1 and 2 interfaces.


Anything that needs "OpenGL", i.e. Wayland/Weston, X11, Chromium, etc etc etc.
These all end up linking to a combination of libmali and mesa-gl.


In general (maybe not for libmali) Wayland and Weston themselves don't
use OpenGL, or mesa-gl. Individual clients may be able to use it,
although it's rare in my (incomplete) experience. Most of the OpenGL
users in Wayland/Weston are going through XWayland/X11 and using GLX.


Weston won't build without a functional OpenGL/libgles configuration.  It errors 
about missing interfaces.  Even with libmali, some of the items need to be 
disabled as they require libgles3, which libmali does not support.




Pretty much anything that depends on 'virtual/gl' (vs virtual/libgles1,2,3).
libmali provides virtual/libgles1 and virtual/libgles2 and a few other things
that meta-gl doesn't.

libmali also doesn't include the DRM/DRI parts.  This comes from mesa-gl.


In general, DRM/DRI doesn't come from mesa-gl. libdrm is its own
recipe, and libgbm comes from mesa proper, or from some other
dedicated vendor specific libgbm recipe.


https://git.yoctoproject.org/poky/tree/meta/recipes-graphics/mesa/mesa-gl_22.2.0.bb

This enables the packageconfig for the gallium set.  Our bbappend enabled the 
dri3 and gallium specifically:


https://github.com/Xilinx/meta-xilinx/blob/master/meta-xilinx-core/recipes-graphics/mesa/mesa-gl_%25.bbappend

This allows the display out for any application using wayland/xwayland.



If this is sidetracking the discussion please ignore; I'm just a
little confused by the above statements because either A) libmali is
really weird B) my understanding of things is _way_ off, or C) the
statements aren't quite correct.


libmali just provides a basic set of interfaces that call the mali co-processor 
to perform actions.  The interfaces happen to be libgles1 and 2 and defined by 
Kronos.  It does NOT provide any way to display content, the DRI/DRM interfaces 
are used for this.


So you end up with a configuration for a system that COULD be as simple as 
libmali + fbdev, where the application renders something with libmali, then uses 
fbdev to display it.  Or you can use X11, or you can use Wayland/Weston.


We have a configuration where Chromium is linked to BOTH mesa-gl and libmali in 
order to do it's rendering, while using Wayland as the compositor/display 
interfaces... which goes through the DRM system.



But ultimately the openGL part is just an engine to "do something", which 
usually involves drawing shapes into memory.  The 'display' side of things 
happens elsewhere.. and this elsewhere CAN be in mesa, it CAN be in wayland, it 
can be fbdev, etc... lots of ways, but unless you are creating all custom apps 
-- you need to be able to use mesa and the existing X11 or Wayland/Weston 
interfaces (via mesa) in order to do this.


--Mark



--Mark


Alex









-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#172173): 
https://lists.openembedded.org/g/openembedded-core/message/172173
Mute This Topic: https://lists.openembedded.org/mt/94420106/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH v2] mesa: Add native patch via a variable

[Mark Hatle]

On 10/26/22 11:03 AM, Alexander Kanavin wrote:

On Wed, 26 Oct 2022 at 16:35, Mark Hatle  wrote:

(I just saw this, so a little late on the reply, but..)

mesa-gl is ABSOLUTELY still being used.  It's needed for libmali usage.  Not
everyone wants to use lima support for graphics.

It was broken into two separate packages so that is was VERY clear if you were
using mesa for 'everything' (mesa), or mesa JUST for 'gl' (not gles).


Okay, I'm not going to propose this, but can you clarify, what is the
current use for 'gl'? Is it just opengl 3d in standalone x server
based systems (which is slowly dying), or is there something else to
it?


Anything that needs "OpenGL", i.e. Wayland/Weston, X11, Chromium, etc etc etc. 
These all end up linking to a combination of libmali and mesa-gl.


Pretty much anything that depends on 'virtual/gl' (vs virtual/libgles1,2,3). 
libmali provides virtual/libgles1 and virtual/libgles2 and a few other things 
that meta-gl doesn't.


libmali also doesn't include the DRM/DRI parts.  This comes from mesa-gl.

--Mark


Alex

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#172165): 
https://lists.openembedded.org/g/openembedded-core/message/172165
Mute This Topic: https://lists.openembedded.org/mt/94420106/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH v2] mesa: Add native patch via a variable

[Mark Hatle]

On 10/19/22 5:35 AM, Alexander Kanavin wrote:

This also begs the question if mesa-gl is actually used by anyone, and
what warrants its continued inclusion in core as opposed to
product/BSP layers. I'll propose dropping the recipe and merging .inc
into mesa.bb once patches start flowing.

The original commit was:
https://git.yoctoproject.org/poky/commit/?h=master-next=015cb13


(I just saw this, so a little late on the reply, but..)

mesa-gl is ABSOLUTELY still being used.  It's needed for libmali usage.  Not 
everyone wants to use lima support for graphics.


It was broken into two separate packages so that is was VERY clear if you were 
using mesa for 'everything' (mesa), or mesa JUST for 'gl' (not gles).


--Mark


Alex

On Wed, 19 Oct 2022 at 08:54, Alexander Kanavin via
lists.openembedded.org 
wrote:


On Wed, 19 Oct 2022 at 08:34, Khem Raj  wrote:


On Tue, Oct 18, 2022 at 10:42 PM Alexander Kanavin
 wrote:


Can you show examples where this is needed? Custom variable does not seem a 
good idea. I also worry that it’ll break automated updates (it’s bad enough 
with mesa-gl, this might make it worse).


https://github.com/ndechesne/meta-qcom/blob/master/recipes-graphics/mesa/mesa_git.bb


"require recipes-graphics/mesa/mesa.inc"

Using recipe includes across layers like this is not a good idea. It
imposes requirements on core to continue providing the .inc, and not
doing anything with mesa that can 'break' users of that include.

Please make it self-contained.

Alex








-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#172160): 
https://lists.openembedded.org/g/openembedded-core/message/172160
Mute This Topic: https://lists.openembedded.org/mt/94420106/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [langdale,master][PATCH] insane.bbclass: Allow hashlib version that only accepts on parameter

[Mark Hatle]

Some versions of hashlib don't appear to implement the second FIPS
related argument.  Detect this and support both versions.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
This was found on an internal Ubuntu 18.04 container.  Unfortunately I
don't have access to the container itself but this resolves the issue.

 meta/classes-global/insane.bbclass | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/meta/classes-global/insane.bbclass 
b/meta/classes-global/insane.bbclass
index db34b4bdb5..dc46857a19 100644
--- a/meta/classes-global/insane.bbclass
+++ b/meta/classes-global/insane.bbclass
@@ -555,7 +555,10 @@ python populate_lic_qa_checksum() {
 import hashlib
 lineno = 0
 license = []
-m = hashlib.new('MD5', usedforsecurity=False)
+try:
+m = hashlib.new('MD5', usedforsecurity=False)
+except TypeError:
+m = hashlib.new('MD5')
 for line in f:
 lineno += 1
 if (lineno >= beginline):
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#171515): 
https://lists.openembedded.org/g/openembedded-core/message/171515
Mute This Topic: https://lists.openembedded.org/mt/94168126/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] wic/wks boot using labels instead of partitions

[Mark Hatle]

Thank you, this worked perfectly.

On 9/17/22 1:20 AM, Markus Volk wrote:

--use-label

This should do it.

Am Fr, 16. Sep 2022 um 16:41:48 -0500 schrieb Mark Hatle 
:
Is there a way to tell the _generated_ fstab to use labels to boot and not 
hard coded partitions? If I remove the --ondisk  attribute from the 
wks file it defaults to /dev/sda. What I tried was: part /boot --source 
bootimg-partition --fstype=vfat --label boot --active --align 4 size 16 part / 
--source rootfs --fstype=ext4 --label root --align 4 If I manually add 
--no-fstab-update to the wks entries and add the following to the system 
default fstab: LABEL=boot /boot vfat default 0 2 This works. But I'd like to 
automate it within the wks. (Alternatively I could use uuid instead of label 
booting, but either case, I need to only boot with the uuid or label -- not 
the partition as the location of the disk can change.) --Mark






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#170836): 
https://lists.openembedded.org/g/openembedded-core/message/170836
Mute This Topic: https://lists.openembedded.org/mt/93732744/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] wic/wks boot using labels instead of partitions

[Mark Hatle]

Is there a way to tell the _generated_ fstab to use labels to boot and not hard 
coded partitions?


If I remove the --ondisk  attribute from the wks file it defaults to 
/dev/sda.


What I tried was:

part /boot --source bootimg-partition --fstype=vfat --label boot --active 
--align 4 size 16

part / --source rootfs --fstype=ext4 --label root --align 4


If I manually add --no-fstab-update to the wks entries and add the following to 
the system default fstab:


LABEL=boot   /boot  vfat  default  0  2


This works.  But I'd like to automate it within the wks.

(Alternatively I could use uuid instead of label booting, but either case, I 
need to only boot with the uuid or label -- not the partition as the location of 
the disk can change.)


--Mark

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#170823): 
https://lists.openembedded.org/g/openembedded-core/message/170823
Mute This Topic: https://lists.openembedded.org/mt/93732744/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [Openembedded-architecture] Adding more information to the SBOM

[Mark Hatle]

On 9/16/22 10:18 AM, Alberto Pianon wrote:

... trimmed ...


I also can see the issue with multiple sources in SRC_URI, although you
should be able to map those back if you assume subtrees are "owned" by
given SRC_URI entries. I suspect there may be a SPDX format limit in
documenting that piece?


I'm replying in reverse order:

- there is a SPDX format limit, but it is by design: a SPDX package
entity is a single sw distribution unit, so it may have only one
downloadLocation; if you have more than one downloadLocation, you must
have more than one SPDX package, according to SPDX specs;


I think my interpretation of this is different.  I've got a view of 'sourcing 
materials', and then verifying the are what we think they are and can be used 
the way we want.  The "upstream sources" (and patches) are really just 'raw 
materials' that we use the Yocto Project to combined to create "the source".


So for the purpose of the SPDX, each upstream source _may_ have a corresponding 
SPDX, but for the binaries their source is the combined unit.. not multiple 
SPDXes.  Think of it something like:


upstream source1 - SPDX
upstream source2 - SPDX
upstream patch
recipe patch1
recipe patch2

In the above, each of those items would be combined by the recipe system to 
construct the source used to build an individual recipe (and collection of 
packages).  Automation _IS_ used to combine the components [unpack/fetch] and 
_MAY_ be used to generated a combined SPDX.


So your "upstream" location for this recipe is the local machine's source 
archive.  The SPDX for the local recipe files can merge the SPDX information 
they know (and if it's at a file level) can use checksums to identify the items 
not captured/modified by the patches for further review (either manual or 
automation like fossology).  In the case where an upstream has SPDX data, you 
should be able to inherit MOST files this way... but the output is specific to 
your configuration and patches.


1 - SPDX |
2 - SPDX |
patch|---> recipe specific SPDX
patch|
patch|

In some cases someone may want to generate SPDX data for the 3 patches, but that 
may or may not be useful in this context.



- I understand that my solution is a bit hacky; but IMHO any other
*post-mortem* solution would be far more hacky; the real solution
would be collecting required information directly in do_fetch and
do_unpack


I've not looked at the current SPDX spec, but past versions has a notes section. 
 Assuming this is still present you can use it to reference back to how this 
component was constructed and the upstream source URIs (and SPDX files) you used 
for processing.


This way nothing really changes in do_fetch or do_unpack.  (You may want to find 
a way to capture file checksums and what the source was for a particular file.. 
but it may not really be necessary!)



- I also understand that we should reduce pain, otherwise nobody would
use our solution; the simplest and cleanest way I can think about is
collecting just package (in the SPDX sense) files' relative paths and
checksums at every stage (fetch, unpack, patch, package), and leave
data processing (i.e. mapping upstream source packages -> recipe's
WORKDIR package -> debug source package -> binary packages -> binary
image) to a separate tool, that may use (just a thought) a graph
database to process things more efficiently.


Even it do_patch nothing really changes, other then again you may want to 
capture checksums to identify thingsthat need further processing.



This approach greatly simplifies things, and gives people doing code reviews the 
insight into what is the source used when shipping the binaries (which is really 
an important aspect of this), as well as which recipe and "build" (really 
fetch/unpack/patch) were used to construct the sources.  If they want to 
investigate the sources further back to their provider, then the notes would 
have the information for that, and you could transition back to the "raw 
materials" providers.




Where I became puzzled is where you say "Information about debug
sources for each actual binary file is then taken from
tmp/pkgdata//extended/*.json.zstd". This is the data we added
and use for the spdx class so you shouldn't need to reinvent that
piece. It should be the exact same data the spdx class uses.



you're right, but in the context of a POC it was easier to extract them
directly from json files than from SPDX data :) It's just a POC to show
that required information may be retrieved in some way, implementation
details do not matter


I was also puzzled about the difference between rpm and the other
package backends. The exact same files are packaged by all the package
backends so the checksums from do_package should be fine.



Here I may miss some piece of information. I looked at files in
tmp/pkgdata but I couldn't find package file checksums anywhere: that is
why I parsed rpm packages. But if 

Re: [OE-core] [Openembedded-architecture] Adding more information to the SBOM

[Mark Hatle]

On 9/14/22 9:56 AM, Joshua Watt wrote:

On Wed, Sep 14, 2022 at 9:16 AM Marta Rybczynska  wrote:


Dear all,
(cross-posting to oe-core and *-architecture)
In the last months, we have worked in Oniro on using the create-spdx
class for both IP compliance and security.

During this work, Alberto Pianon has found that some information is
missing from the SBOM and it does not contain enough for Software
Composition Analysis. The main missing point is the relation between
the actual upstream sources and the final binaries (create-spdx uses
composite sources).


I believe we map the binaries to the source code from the -dbg
packages; is the premise that this is insufficient? Can you elaborate
more on why that is, I don't quite understand. The debug sources are
(basically) what we actually compiled (e.g. post-do_patch) to produce
the binary, and you can in turn follow these back to the upstream
sources with the downloadLocation property.


When I last looked at this, it was critical that the analysis be:

binary -> patched & configured source (dbg package) -> how the sources were 
constructed.


As Joshua said above.  I believe all of the information is present for this as 
you can tie the binary (through debug symbols) back to the debug package.. and 
the source of the debug package back to the sources that constructed it via 
heuristics.  (If you enable the git patch mechanism.  It should even be possible 
to use git blame to find exactly what upstreams constructed the patched sources.


For generated content, it's more difficult -- but for those items usually there 
is a header which indicates what generated the content so other heuristics can 
be used.




Alberto has worked on how to obtain the missing data and now has a
POC. This POC provides full source-to-binary tracking of Yocto builds
through a couple of scripts (intended to be transformed into a new
bbclass at a later stage). The goal is to add the missing pieces of
information in order to get a "real" SBOM from Yocto, which should, at
a minimum:


Please be a little careful with the wording; SBoMs have a lot of uses,
and many of them we can satisfy with what we currently generate; it
may not do the exact use case you are looking for, but that doesn't
mean it's not a "real" SBoM :)



- carefully describe what is found in a final image (i.e. binary files
and their dependencies), since that is what is actually distributed
and goes into the final product;
- describe how such binary files have been generated and where they
come from (i.e. upstream sources, including patches and other stuff
added from meta-layers); provenance is important for a number of
reasons related to IP Compliance and security.


Full compliance will require binaries mapped to patched source to upstream 
sources _AND_ the instructions (layer/recipe/configuration) used to build them. 
 But it's up to the local legal determination to figure out 'how far you really 
need to go', vs just "here are the layers I used to build my project".)



The aim is to become able to:

- map binaries to their corresponding upstream source packages (and
not to the "internal" source packages created by recipes by combining
multiple upstream sources and patches)
- map binaries to the source files that have been actually used to
build them - which usually are a small subset of the whole source
package

With respect to IP compliance, this would allow to, among other things:

- get the real license text for each binary file, by getting the
license of the specific source files it has been generated from
(provided by Fossology, for instance), - and not the main license
stated in the corresponding recipe (which may be as confusing as
GPL-2.0-or-later & LGPL-2.1-or-later & BSD-3-Clause & BSD-4-Clause, or
even worse)


IIUC this is the difference between the "Declared" license and the
"Concluded" license. You can report both, and I think
create-spdx.bbclass can currently do this with its rudimentary source
license scanning. You really do want both and it's a great way to make
sure that the "Declared" license (that is the license in the recipe)
reflects the reality of the source code.


And the thing to keep in mind is that in a given package the "Declared" is 
usually what a LICENSE file or header says.  But the "Concluded" has levels of 
quality behind them.  The first level of quality is "Declared".  The next level 
is automation (something like fossology), the next level is human reviewed, and 
the highest level is "lawyer reviewed".


So being able to inject SPDX information with Concluded values for evaluation 
and track the 'quality level' has always been something I wanted to do, but 
never had time.


At the time, my idea was a database (and/or bbappend) for each component that 
would included pre-processed SPDX data for each recipe.  This data would run 
through a validation step to show it actually matches the patched sources.  (If 
any file checksums do NOT match, then they would be flagged for 

Re: [OE-core] [Openembedded-architecture] Configuration fragments

[Mark Hatle]

Wind River already has a mechanism to do something like this, called templates.

https://github.com/WindRiver-Labs/wr-template/tree/WRLINUX_10_21_BASE

Each template can have (optionally):

#   README - template README file
#   require - list of other templates required for this one
#   template.conf - template configuration fragment
#   image.inc - image fragment
#   bsp-pkgs.conf - BSP specific configuration fragment
#   bsp-pkgs.inc - BSP specific image fragment

# The 'bsp-pkgs' files can only be in a template in a layer that provides a
# specific conf/machine/${MACHINE}.conf file and layers it may contain,
# otherwise they will be ignored

I'm not saying what is implemented is a perfect solution, but it made a 
'configuration fragment' approach to system configuration much easier when I 
worked with it.


(For examples see: 
https://github.com/WindRiver-Labs/wrlinux/tree/WRLINUX_10_21_BASE/templates/feature)


You could do things (in your local.conf, or machine.conf or distro.conf) like:

WRTEMPLATE += "dpdk"

This would add the dpdk recipes to your image and also configure the kernel for 
dpdk.


https://github.com/WindRiver-Labs/wrlinux/tree/WRLINUX_10_21_BASE/templates/feature/dpdk

--Mark

On 9/1/22 10:29 AM, Alexander Kanavin wrote:

On Thu, 18 Aug 2022 at 11:27, Richard Purdie
 wrote:

The intent is the user could add something like:

require conf/yocto-autobuilder/x32.inc

or

require conf/yocto-autobuilder/multilib-mipsn32.inc

and similar to their local.conf and more easily replicate
configurations.


This could be a subset of a broader feature, one that allows adding
and removing pre-canned 'config fragments' to local.conf. The proposal
is to have such fragments live in
meta-somelayer/conf/fragments/path/to/some/fragment (I think allowing
multiple levels is beneficial here), and each fragment would start
with a comment explaining what it does followed by the actual
settings.

Then adding fragments can be done by appending them to local.conf, and
removing by looking for markers in local.conf and editing it out,
which probably could use a helper tool/subcommand.

Layers could also declare 'non-optional' fragments which get included
automatically by the fact of including the layer into the build, which
means layer.conf can go back to being short and sweet and to the
point.

Thoughts?


I would like to see if we can switch the eSDK to use the json format
btw. I suspect that may be one of your next steps? :)


I don't think I understand this, can you explain?

Alex






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#170229): 
https://lists.openembedded.org/g/openembedded-core/message/170229
Mute This Topic: https://lists.openembedded.org/mt/93407466/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] rootfs-postcommands: Make /etc/timestamp consistent with image

[Mark Hatle]

On 8/29/22 6:59 PM, William A. Kennington III via lists.openembedded.org wrote:

This makes the determination of the timestamp for the /etc/timestamp
file consistent with mtimes in the generated image. This is desirable to
make the built image reproducible with the git commit date instead of
the current date.


This is only going to pay attention to poky, or oe-core...  the rootfs will vary 
with a change to ANY layer in the system.


If you really want to be consistent you would need to iterate over all of the 
layers and look at every file in the system for the timestamp (or if they're git 
repositories last commit in each layer..)


All of this doesn't seem like it really makes sense to me which is why a static 
timestamp was defined previously.


(with the previous change to the time stamps, I've had more then one user get 
concerned they were 'using old software' when they just built something from 
scratch -- this will get even more confusing if it's only based on the 
oe-core/poky repository.)


--Mark


Change-Id: I7d9fe32906aa93baf53948aa40b7a98fb05dd384
Signed-off-by: William A. Kennington III 
---
  meta/classes-recipe/rootfs-postcommands.bbclass | 12 +++-
  1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/meta/classes-recipe/rootfs-postcommands.bbclass 
b/meta/classes-recipe/rootfs-postcommands.bbclass
index 215e38e33d..8d710186d7 100644
--- a/meta/classes-recipe/rootfs-postcommands.bbclass
+++ b/meta/classes-recipe/rootfs-postcommands.bbclass
@@ -312,12 +312,14 @@ python write_image_manifest () {
  # Can be used to create /etc/timestamp during image construction to give a 
reasonably
  # sane default time setting
  rootfs_update_timestamp () {
-   if [ "${REPRODUCIBLE_TIMESTAMP_ROOTFS}" != "" ]; then
-   # Convert UTC into %4Y%2m%2d%2H%2M%2S
-   sformatted=`date -u -d @${REPRODUCIBLE_TIMESTAMP_ROOTFS} 
+%4Y%2m%2d%2H%2M%2S`
-   else
-   sformatted=`date -u +%4Y%2m%2d%2H%2M%2S`
+   if [ "$REPRODUCIBLE_TIMESTAMP_ROOTFS" = "" ]; then
+   REPRODUCIBLE_TIMESTAMP_ROOTFS=`git -C "${COREBASE}" log -1 
--pretty=%ct 2>/dev/null` || true
+   if [ "$REPRODUCIBLE_TIMESTAMP_ROOTFS" = "" ]; then
+   REPRODUCIBLE_TIMESTAMP_ROOTFS=`stat -c%Y 
${@bb.utils.which(d.getVar("BBPATH"), "conf/bitbake.conf")}`
+   fi
fi
+   # Convert UTC into %4Y%2m%2d%2H%2M%2S
+   sformatted=`date -u -d @${REPRODUCIBLE_TIMESTAMP_ROOTFS} 
+%4Y%2m%2d%2H%2M%2S`
echo $sformatted > ${IMAGE_ROOTFS}/etc/timestamp
bbnote "rootfs_update_timestamp: set /etc/timestamp to $sformatted"
  }






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#170042): 
https://lists.openembedded.org/g/openembedded-core/message/170042
Mute This Topic: https://lists.openembedded.org/mt/93339153/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] pseudo deadlock in rust via malloc backtrace

[Mark Hatle]

On 8/4/22 11:27 AM, Richard Purdie wrote:

On Thu, 2022-08-04 at 12:36 +0100, Khem Raj wrote:



On Thu, Aug 4, 2022 at 10:05 AM Richard Purdie
 wrote:

Just to note that where we've been seeing pseudo hangs in rust, it
seems jemalloc calls obtain_malloc_conf() which calls readlink()
which
pseudo intercepts and then calls malloc() which loops back to
jemalloc
which deadlocks. What fun.



(Going way back) the way this was avoided in the distant past was having a 
static variable of whatever the 'max size' was already allocated in a few key 
locations and using that variable instead of a malloc.  (Of course due to it 
having a fixed length a size check and such needed to be done.)


I don't really know the context at this point for this item, but it's an 
alternative way to solve this problem, but will require memory to be allocated 
at compile time, either in a function or a small global pool for usage.


It does make me wonder if there are other function in the system that use malloc 
and if so could they use an alternative, if not the 'is_init' might be the only 
way to handle it.


--Mark



I guess pseudo is doing this extra malloc right ?


Yes


  Maybe disable instrumentation in jemalloc when building non target
packages. This might avoid the issue at least



Sadly it doesn't help as it wants to load it isn't configuration and
the system setup from /proc too.

Cheers,

Richard






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#168976): 
https://lists.openembedded.org/g/openembedded-core/message/168976
Mute This Topic: https://lists.openembedded.org/mt/92809824/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH 4/4] elfutils: Microblaze does not support symvers

[Mark Hatle]

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/recipes-devtools/elfutils/elfutils_0.187.bb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/meta/recipes-devtools/elfutils/elfutils_0.187.bb 
b/meta/recipes-devtools/elfutils/elfutils_0.187.bb
index d03da61353..561112c580 100644
--- a/meta/recipes-devtools/elfutils/elfutils_0.187.bb
+++ b/meta/recipes-devtools/elfutils/elfutils_0.187.bb
@@ -49,6 +49,9 @@ RDEPENDS:${PN}-ptest += "libasm libelf bash make coreutils 
${PN}-binutils iprout
 
 EXTRA_OECONF:append:class-target = " --disable-tests-rpath"
 
+# symver functions not currently supported on microblaze
+EXTRA_OECONF:append:class-target:microblaze = " --disable-symbol-versioning"
+
 RDEPENDS:${PN}-ptest:append:libc-glibc = " glibc-utils glibc-dbg glibc-dev"
 INSANE_SKIP:${PN}-ptest = "debug-deps dev-deps"
 
-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#168522): 
https://lists.openembedded.org/g/openembedded-core/message/168522
Mute This Topic: https://lists.openembedded.org/mt/92639282/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH 3/4] openssl: Move microblaze to linux-latomic config

[Mark Hatle]

When building with the previous a number of atomic functions come back as
undefined.  Switching to linux-latomic fixes this.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/recipes-connectivity/openssl/openssl_3.0.5.bb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/recipes-connectivity/openssl/openssl_3.0.5.bb 
b/meta/recipes-connectivity/openssl/openssl_3.0.5.bb
index e50ff7f8c5..04aff04fab 100644
--- a/meta/recipes-connectivity/openssl/openssl_3.0.5.bb
+++ b/meta/recipes-connectivity/openssl/openssl_3.0.5.bb
@@ -77,7 +77,7 @@ do_configure () {
esac
target="$os-${HOST_ARCH}"
case $target in
-   linux-arc)
+   linux-arc | linux-microblaze*)
target=linux-latomic
;;
linux-arm*)
@@ -105,7 +105,7 @@ do_configure () {
linux-*-mips64 | linux-mips64 | linux-*-mips64el | linux-mips64el)
target=linux64-mips64
;;
-   linux-microblaze* | linux-nios2* | linux-sh3 | linux-sh4 | linux-arc*)
+   linux-nios2* | linux-sh3 | linux-sh4 | linux-arc*)
target=linux-generic32
;;
linux-powerpc)
-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#168521): 
https://lists.openembedded.org/g/openembedded-core/message/168521
Mute This Topic: https://lists.openembedded.org/mt/92639281/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH 0/4] Various fixes for master

[Mark Hatle]

While working on forward porting microblaze support I ran into the following 
patches.

default-distrovars.inc change, we've been doing this in our petalinux distro
-- but it seems like a better idea to do it generically.

The runqemu one is a missing space in code that I THINK is only used by
microblaze systems currently.  (Or at least Xilinx systems with the custom
qemu-xilinx.)

The other two are simple recipe updates.

Mark Hatle (4):
  runqemu: Add missing space on default display option
  default-distrovars: seccomp doesn't support microblaze
  openssl: Move microblaze to linux-latomic config
  elfutils: Microblaze does not support symvers

 meta/conf/distro/include/default-distrovars.inc| 3 +++
 meta/recipes-connectivity/openssl/openssl_3.0.5.bb | 4 ++--
 meta/recipes-devtools/elfutils/elfutils_0.187.bb   | 3 +++
 scripts/runqemu| 2 +-
 4 files changed, 9 insertions(+), 3 deletions(-)

-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#168518): 
https://lists.openembedded.org/g/openembedded-core/message/168518
Mute This Topic: https://lists.openembedded.org/mt/92639278/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH 1/4] runqemu: Add missing space on default display option

[Mark Hatle]

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 scripts/runqemu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/runqemu b/scripts/runqemu
index b4c1ae6d83..b6fc212ebe 100755
--- a/scripts/runqemu
+++ b/scripts/runqemu
@@ -1375,7 +1375,7 @@ class BaseConfig(object):
 elif "-display sdl" in output:
 self.sdl = True
 else:
-self.qemu_opt += '-display none'
+self.qemu_opt += ' -display none'
 
 if self.sdl == True or self.gtk == True or self.egl_headless == True:
 
-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#168520): 
https://lists.openembedded.org/g/openembedded-core/message/168520
Mute This Topic: https://lists.openembedded.org/mt/92639280/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH 2/4] default-distrovars: seccomp doesn't support microblaze

[Mark Hatle]

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/conf/distro/include/default-distrovars.inc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/meta/conf/distro/include/default-distrovars.inc 
b/meta/conf/distro/include/default-distrovars.inc
index 9f4617be01..230bab84dd 100644
--- a/meta/conf/distro/include/default-distrovars.inc
+++ b/meta/conf/distro/include/default-distrovars.inc
@@ -16,6 +16,9 @@ DISTRO_FEATURES_DEFAULT:remove:riscv32 = "seccomp"
 # seccomp is not yet ported to ARC
 DISTRO_FEATURES_DEFAULT:remove:arc = "seccomp"
 
+# seccomp is not yet ported to microblaze
+DISTRO_FEATURES_DEFAULT:remove:microblaze = "seccomp"
+
 DISTRO_FEATURES_DEFAULT ?= "acl alsa bluetooth debuginfod ext2 ipv4 ipv6 
largefile pcmcia usbgadget usbhost wifi xattr nfs zeroconf pci 3g nfc x11 vfat 
seccomp"
 DISTRO_FEATURES ?= "${DISTRO_FEATURES_DEFAULT}"
 IMAGE_FEATURES ?= ""
-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#168519): 
https://lists.openembedded.org/g/openembedded-core/message/168519
Mute This Topic: https://lists.openembedded.org/mt/92639279/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] sstate: inside the threadedpool don't write to the shared localdata

[Mark Hatle]

Note, it appears this bug is in honister as well.

On 3/6/22 3:08 PM, Jose Quaresma wrote:

When inside the threadedpool we make a copy of the localdata
to avoid some race condition, so we need to use this new
localdata2 and stop write the shared localdata.

Signed-off-by: Jose Quaresma 
---
  meta/classes/sstate.bbclass | 2 +-
  1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index dc9a2c085b..7aca415159 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -988,7 +988,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, 
currentcount=0, summary=True,
  
  localdata2 = bb.data.createCopy(localdata)

  srcuri = "file://" + sstatefile
-localdata.setVar('SRC_URI', srcuri)
+localdata2.setVar('SRC_URI', srcuri)
  bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
  
  import traceback







-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#162853): 
https://lists.openembedded.org/g/openembedded-core/message/162853
Mute This Topic: https://lists.openembedded.org/mt/89597961/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH 1/1] insane.bbclass: Update insane.bbclass to work on FIPS enabled hosts

[Mark Hatle]

hashlib.md5() is not permitted on a FIPS enabled host system.  This is due
to md5 not being an approved hash algorithm.

Instead use:
 hashlib.new('MD5', usedforsecurity=False)

This is allowed, as it's clear the hash is used for a non-security purpose.

Using an md5 to identify when a license has changed is permitted, as we're
not using it for file integrity.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/classes/insane.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes/insane.bbclass b/meta/classes/insane.bbclass
index 890e865a8f..29b9b3d466 100644
--- a/meta/classes/insane.bbclass
+++ b/meta/classes/insane.bbclass
@@ -549,7 +549,7 @@ python populate_lic_qa_checksum() {
 import hashlib
 lineno = 0
 license = []
-m = hashlib.md5()
+m = hashlib.new('MD5', usedforsecurity=False)
 for line in f:
 lineno += 1
 if (lineno >= beginline):
-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#162520): 
https://lists.openembedded.org/g/openembedded-core/message/162520
Mute This Topic: https://lists.openembedded.org/mt/8940/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH 0/1] FIPS host support

[Mark Hatle]

The patch here, and one sent to bitbake-devel together enable basic support
for a FIPS-140 host system.

What was identified were a few users of md5, which is not allowed for any
security part of the system.  It can still be used to identify changes and
similar non-security activities.  (OE already uses sha256 for file
integrity.)

In addition to this, it's possible that a recipe may attempt to use md5
during the build process.  In oe-core, the only user is 'ovmf'.  At this
time I don't intend to provide a fix for ovmf, but everything else in core
works properly now.

Mark Hatle (1):
  insane.bbclass: Update insane.bbclass to work on FIPS enabled hosts

 meta/classes/insane.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

-- 
2.25.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#162521): 
https://lists.openembedded.org/g/openembedded-core/message/162521
Mute This Topic: https://lists.openembedded.org/mt/8941/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH v2 3/3] systemd: Minimize udev package size if DISTRO_FEATURES contains systemd

[Mark Hatle]

On 2/6/22 11:35 AM, Richard Purdie wrote:

On Sun, 2022-02-06 at 18:31 +0100, Alexander Kanavin wrote:

On Sun, 6 Feb 2022 at 18:27, Mark Hatle 
wrote:

It definitely works in Honister (I'm actively using it.)

We produce a single package set, then allow for multiple image recipes to
select
which init manager to use.  This simplifies the eSDK, sstate-cache and
binary
package feed -- while still giving the end user the ability to switch.




Selection of init manager is a distro setting, it can be used by component
recipes to make decisions, and overriding it from images is neither tested nor
supported.


There is a hybrid where you can enable sysvinit and systemd at the same time and
use them in different situations. We do test that and it is that test which
showed issues on the autobuilder with this patch.

It isn't as easy as pick and chose either in a given image but handy for
initramfs and some other uses.


I've got the following in my distro:

# Create packages that support both systemd and sysvinit
# but only on arm/arm64 systems, microblaze is sysvinit only
DISTRO_FEATURES_BACKFILL_CONSIDERED:remove:arm = "sysvinit systemd"
DISTRO_FEATURES_BACKFILL_CONSIDERED:remove:aarch64 = "sysvinit systemd"
DISTRO_FEATURES_BACKFILL:append:arm = " systemd"
DISTRO_FEATURES_BACKFILL:append:aarch64 = " systemd”

INIT_MANAGER_DEFAULT = "systemd"
INIT_MANAGER_DEFAULT:microblaze = "sysvinit"
INIT_MANAGER_DEFAULT:zynq = "sysvinit"

INIT_MANAGER ?= "${INIT_MANAGER_DEFAULT}"

Then a few configurations change the INIT_MANAGER, and (almost) all of the 
sstate is re-usable and things don't need to be re-built.


The issue I have, microblaze if the filesystem is over about 80mb it doesn't fit 
into memory.  On Arm, some devices are memory limited the same way microblaze 
is, while some have full access to flash storage and thus can use systemd.  And 
on aarch64 - most defined (but not all) have flash storage.


This is all about sstate-cache re-use (time to build the image) and ability to 
have a SINGLE binary distribution that supports the image size requirements of 
the user.


--Mark


Cheers,

Richard








-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161410): 
https://lists.openembedded.org/g/openembedded-core/message/161410
Mute This Topic: https://lists.openembedded.org/mt/88853822/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH v2 3/3] systemd: Minimize udev package size if DISTRO_FEATURES contains systemd

[Mark Hatle]

On 2/6/22 10:39 AM, Stefan Herbrechtsmeier wrote:

Am 06.02.22 um 09:10 schrieb Richard Purdie:

On Wed, 2022-02-02 at 08:35 +0100, Stefan Herbrechtsmeier wrote:

From: Stefan Herbrechtsmeier 

Link udev shared with systemd helper to minimize the udev package size
if DISTRO_FEATURES contains systemd.

It is only usefull to link udev static with systemd helper if udev
should be installed without systemd.

Signed-off-by: Stefan Herbrechtsmeier 

---

(no changes since v1)

   meta/recipes-core/systemd/systemd_249.7.bb | 1 +
   1 file changed, 1 insertion(+)



This seems to break some of our tests:

https://autobuilder.yoctoproject.org/typhoon/#/builders/72/builds/4708
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/3121
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/3085
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/3089
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/3127

Presumably it causes extra dependencies to be pulled in causing the file overlap
where there was previously none.


Sorry, I was told sysvinit with systemd isn't a valid use case anymore.


It definitely works in Honister (I'm actively using it.)

We produce a single package set, then allow for multiple image recipes to select 
which init manager to use.  This simplifies the eSDK, sstate-cache and binary 
package feed -- while still giving the end user the ability to switch.


(And there is a HUGE difference in image size between sysvinit and systemd.)

--Mark


Would it be okay to enable it only if sysvinit isn't in distro?

Otherwise reject the change because it is impossible to know if a
dependency between udev and systemd is okay or not.

Regads
Stefan






-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161406): 
https://lists.openembedded.org/g/openembedded-core/message/161406
Mute This Topic: https://lists.openembedded.org/mt/88853822/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH v2 3/3] systemd: Minimize udev package size if DISTRO_FEATURES contains systemd

[Mark Hatle]

I just noticed this.  It might make sense to link udev static when BOTH systemd 
and sysvinit are enabled.  (A single package set, multiple images with different 
init managers.)


Not sure though how that works in Honister, just something I noticed with this 
commit.  (And yes, I do use a distro with BOTH sysvinit and systemd enabled for 
a single package set.)


--Mark

On 2/6/22 2:10 AM, Richard Purdie wrote:

On Wed, 2022-02-02 at 08:35 +0100, Stefan Herbrechtsmeier wrote:

From: Stefan Herbrechtsmeier 

Link udev shared with systemd helper to minimize the udev package size
if DISTRO_FEATURES contains systemd.

It is only usefull to link udev static with systemd helper if udev
should be installed without systemd.

Signed-off-by: Stefan Herbrechtsmeier 

---

(no changes since v1)

  meta/recipes-core/systemd/systemd_249.7.bb | 1 +
  1 file changed, 1 insertion(+)



This seems to break some of our tests:

https://autobuilder.yoctoproject.org/typhoon/#/builders/72/builds/4708
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/3121
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/3085
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/3089
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/3127

Presumably it causes extra dependencies to be pulled in causing the file overlap
where there was previously none.

Cheers,

Richard








-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161405): 
https://lists.openembedded.org/g/openembedded-core/message/161405
Mute This Topic: https://lists.openembedded.org/mt/88853822/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] sanity: Drop TUNEABI, TUNEABI_WHITELIST, TUNEABI_OVERRIDE

[Mark Hatle]

Just as confirmation, places I know this code was used are no longer being used.

(Toolchains now pretty much all come from YP sources vs a magic binary)

--Mark

On 1/13/22 3:27 PM, Richard Purdie wrote:

These were added nearly a decade ago but there are no users in OE-Core. I
checked with the likely users and they seem to have no current usage either.
Therefore remove them.

If needed for some prebuilt library somewhere, they could be implemented
in the layer using them instead but I doubt these are in use any longer.

Signed-off-by: Richard Purdie 
---
  meta/classes/sanity.bbclass  | 42 
  meta/conf/documentation.conf |  3 ---
  2 files changed, 45 deletions(-)

diff --git a/meta/classes/sanity.bbclass b/meta/classes/sanity.bbclass
index 0e20589b22d..f288b4c84c9 100644
--- a/meta/classes/sanity.bbclass
+++ b/meta/classes/sanity.bbclass
@@ -185,37 +185,6 @@ def raise_sanity_error(msg, d, network_error=False):
  
  %s""" % msg)
  
-# Check flags associated with a tuning.

-def check_toolchain_tune_args(data, tune, multilib, errs):
-found_errors = False
-if check_toolchain_args_present(data, tune, multilib, errs, 'CCARGS'):
-found_errors = True
-if check_toolchain_args_present(data, tune, multilib, errs, 'ASARGS'):
-found_errors = True
-if check_toolchain_args_present(data, tune, multilib, errs, 'LDARGS'):
-found_errors = True
-
-return found_errors
-
-def check_toolchain_args_present(data, tune, multilib, tune_errors, which):
-args_set = (data.getVar("TUNE_%s" % which) or "").split()
-args_wanted = (data.getVar("TUNEABI_REQUIRED_%s:tune-%s" % (which, tune)) or 
"").split()
-args_missing = []
-
-# If no args are listed/required, we are done.
-if not args_wanted:
-return
-for arg in args_wanted:
-if arg not in args_set:
-args_missing.append(arg)
-
-found_errors = False
-if args_missing:
-found_errors = True
-tune_errors.append("TUNEABI for %s requires '%s' in TUNE_%s (%s)." %
-   (tune, ' '.join(args_missing), which, ' 
'.join(args_set)))
-return found_errors
-
  # Check a single tune for validity.
  def check_toolchain_tune(data, tune, multilib):
  tune_errors = []
@@ -247,17 +216,6 @@ def check_toolchain_tune(data, tune, multilib):
  bb.debug(2, "  %s: %s" % (feature, valid_tunes[feature]))
  else:
  tune_errors.append("Feature '%s' is not defined." % feature)
-whitelist = localdata.getVar("TUNEABI_WHITELIST")
-if whitelist:
-tuneabi = localdata.getVar("TUNEABI:tune-%s" % tune)
-if not tuneabi:
-tuneabi = tune
-if True not in [x in whitelist.split() for x in tuneabi.split()]:
-tune_errors.append("Tuning '%s' (%s) cannot be used with any supported 
tuning/ABI." %
-(tune, tuneabi))
-else:
-if not check_toolchain_tune_args(localdata, tuneabi, multilib, 
tune_errors):
-bb.debug(2, "Sanity check: Compiler args OK for %s." % tune)
  if tune_errors:
  return "Tuning '%s' has the following errors:\n" % tune + 
'\n'.join(tune_errors)
  
diff --git a/meta/conf/documentation.conf b/meta/conf/documentation.conf

index f63f4b223a1..6b50ad08a8b 100644
--- a/meta/conf/documentation.conf
+++ b/meta/conf/documentation.conf
@@ -440,9 +440,6 @@ TOOLCHAIN_TARGET_TASK[doc] = "This variable lists packages 
the OpenEmbedded buil
  TOPDIR[doc] = "The Build Directory. BitBake automatically sets this variable. The 
OpenEmbedded build system uses the Build Directory when building images."
  TRANSLATED_TARGET_ARCH[doc] = "A sanitized version of TARGET_ARCH. This variable 
is used where the architecture is needed in a value where underscores are not 
allowed."
  TUNE_PKGARCH[doc] = "The package architecture understood by the packaging system 
to define the architecture, ABI, and tuning of output packages."
-TUNEABI[doc] = "An underlying ABI used by a particular tuning in a given toolchain 
layer. This feature allows providers using prebuilt libraries to check compatibility of a 
tuning against their selection of libraries."
-TUNEABI_OVERRIDE[doc] = "If set, ignores TUNEABI_WHITELIST."
-TUNEABI_WHITELIST[doc] = "A whitelist of permissible TUNEABI values.  If the 
variable is not set, all values are allowed."
  TUNECONFLICTS[doc] = "List of conflicting features for a given feature."
  TUNEVALID[doc] = "Descriptions, stored as flags, of valid tuning features."
  







-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160533): 
https://lists.openembedded.org/g/openembedded-core/message/160533
Mute This Topic: https://lists.openembedded.org/mt/88407256/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] Yocto : Need help in building latest openembedded branch code

[Mark Hatle]

The variable syntax has changed between older and newer versions of bitbake.

See: 
https://docs.yoctoproject.org/migration-guides/migration-3.4.html#override-syntax-changes


On 11/15/21 11:59 PM, mohammed aqdam wrote:

Hi There,

I am working on a yocto image for the imx8 board, for this I have created a 
project with zeus layers for all layers(including meta-imx) using the below 
manifest file.


|repo init -u https:||//source||.codeaurora.org
||/external/imx/imx-manifest||-b imx-linux-zeus -m
imx-5.4.47-2.2.0.xml|


I am trying to fetch master/honister code of openEmbedded on my 
existing project(Earlier i was using Zeus branch) to use latest modules of 
perl(5.34.0), chrony(4.1) & gpsd(gpsd 3.23.1 
), by modifying 
.repo/manifests/imx-5.4.47-2.2.0.xml with master/honister code for 
openEmbedded(revision="refs/heads/honister"), later i ran into below error,


$ bitbake -k imx-image-full
ERROR: Unable to start bitbake server (None)
ERROR: Server log for this session
(/data/home/maqdam/imx_yocto_master/build/bitbake-cookerdaemon.log):
--- Starting bitbake server pid 6315 at 2021-11-16 10:36:44.095759 ---
ERROR: ParseError at

/data/home/maqdam/imx_yocto_master/sources/meta-openembedded/meta-oe/conf/layer.conf:106:
unparsed line: 'DEFAULT_TEST_SUITES:pn-meta-oe-ptest-image = "
${PTESTTESTSUITE}"'


Tried updating the bitbake layer(poky) to master/honister in similar fashion, 
Later seeing the below error in meta-imx/meta-bsp/conf/layer.conf file.


$ bitbake -k imx-image-full
ERROR: Variable FSL_EULA_FILE_MD5SUMS_append contains an operation using the
old override syntax. Please convert this layer/metadata before attempting to
use with a newer bitbake.


I could not find how to use FSL_EULA_FILE_MD5SUMS_append with the latest 
bitbake.
Request you to suggest how can i use this master/honister code of openEmbedded & 
with which version of bitbake.


Thanks,
Aqdam





-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#158345): 
https://lists.openembedded.org/g/openembedded-core/message/158345
Mute This Topic: https://lists.openembedded.org/mt/87089467/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] openssh: openssh-dev shouldn't depend on openssh

[Mark Hatle]

On 9/29/21 3:36 AM, Matt Johnston wrote:
> On Wed, 2021-09-29 at 09:24 +0100, Richard Purdie wrote:
>>> +RDEPENDS:${PN}-dev = ""
>>
>> At that point what is the point of the -dev package? I think you could make
>> this argument about a lot of the -dev packages and I'm not sure I'd want to
>> see every recipe doing this.
>>
>> What are you using the -dev package for and do you need it at all?
> 
> The -dev package isn't needed for anything, but it gets pulled in when
> "populate_sdk" is built. That does a glob over all -dev packages for
> any built package.

The dev package's SHOULD depend on the regular package by default.  There are
VERY VERY few instances where the dev package is "useful" without the
corresponding run-time package in a 'normal' use-case.  (There are always
exceptions and corner cases, but they're just that exceptions and corner cases
in my experience and lead to massive confusion by developers if exploited.)

populate_sdk, the entire purpose of this is to install the -dev package (and
it's dependencies) for anything in your root filesystem.  This says that you
root filesystem contains "openssh", thus openssh-dev will be installed.

If your root filesystem does NOT have openssh in it, then we need to answer the
question why was the -dev version added?

You mention above it's "any built package", but that should not be the case.  It
should be for any package installed into the corresponding root filesystem.

i.e. bitbake core-image-minimal -c do_populate_sdk

This should NOT install -dev packages for things not located in
core-image-minimal.  If it does, then there is another problem, that was not the
intended design.  The intention when the do_populate_sdk was written was
construct the rootfs, parse what was install and add -dev packages + their
dependencies.

--Mark

> Cheers,
> Matt
> 
>>
>> [As an aside, I do think the -dev package dependencies need rethinking but
>> that is a lot more work which nobody seems very interested in]
>>
>> Cheers,
>>
>> Richard
>>
> 
> 
> 
> 
> 
> 

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156456): 
https://lists.openembedded.org/g/openembedded-core/message/156456
Mute This Topic: https://lists.openembedded.org/mt/85941131/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] user/group XXX does not exist, using root with RPM/DNF packaging in Hardknott and Honister

[Mark Hatle]

On 9/24/21 9:02 AM, Zoltan Boszormenyi via lists.openembedded.org wrote:
> Hi,
> 
> I have a special package that creates users and groups
> via inherit useradd. This package doesn't depend on any
> others but it is depended on, both via DEPENDS and RDEPENDS
> by packages using those users/groups in their do_install
> scripts.
> 
> This works for packaging becase these ownerships
> are encoded in the packages, confirmed by rpm -qp --dump ...

Does it show the useradd in the _PREINSTALL_ (you can use --scriptlets in the
rpm -qp)?

> However, during do_rootfs, a couple of
> "user/group XXX does not exist, using root"
> messages appear for the packages depending on others
> creating these users/groups.

Do the using packages contain RDEPENDS on the package that adds the user/group
to the system?

> log.do_rootfs shows that the package installation ordering
> does not follow RDEPENDS. Instead, it's practically an
> alphabetical order when running dnf.
> 
> This doesn't just involve my custom packages, but also clamav
> plus another one in which I ship a small limited set of
> virus signatures, also chown'd to clamav and with RDEPENDS
> on clamav.
> 
> What is the correct solution to this?

Typically the combination of the pre-install scriptlet, along with RDEPENDS will
ensure that the user has been added before the install completes.

--Mark

> Thanks in advance,
> Zoltán Böszörményi
> 
> 
> 
> 
> 

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156337): 
https://lists.openembedded.org/g/openembedded-core/message/156337
Mute This Topic: https://lists.openembedded.org/mt/85839631/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH] tcf-agent: Move to the latest master version

[Mark Hatle]

There has not been a release since 2018, the 1.7.0 release.  A number of
recent improvements around thumb and clang debugging prompted this move
to a newer version.

The patch is no longer necessary as it was a backport patch.

Signed-off-by: Mark Hatle 
---
 .../0001-Fixed-copyright-messages.patch   | 56 ---
 .../tcf-agent/tcf-agent_git.bb|  3 +-
 2 files changed, 1 insertion(+), 58 deletions(-)
 delete mode 100644 
meta/recipes-devtools/tcf-agent/tcf-agent/0001-Fixed-copyright-messages.patch

diff --git 
a/meta/recipes-devtools/tcf-agent/tcf-agent/0001-Fixed-copyright-messages.patch 
b/meta/recipes-devtools/tcf-agent/tcf-agent/0001-Fixed-copyright-messages.patch
deleted file mode 100644
index ed73808e990..000
--- 
a/meta/recipes-devtools/tcf-agent/tcf-agent/0001-Fixed-copyright-messages.patch
+++ /dev/null
@@ -1,56 +0,0 @@
-From 20ac1f939a8a97b03abec55d865050fdaa0f343a Mon Sep 17 00:00:00 2001
-From: Eugene Tarassov 
-Date: Tue, 8 Jan 2019 21:56:16 -0800
-Subject: [oe-core][PATCH 1/1] Fixed copyright messages
-
-Upstream-Status: Backport 
[https://git.eclipse.org/c/tcf/org.eclipse.tcf.agent.git/commit/?id=20ac1f939a8a97b03abec55d865050fdaa0f343a]
-
-Signed-off-by: Joe Slater 
-Signed-off-by: Mingli Yu 

- agent/tcf/framework/channel_lws.c  |  2 +-
- agent/tcf/framework/cpudefs-mdep-mux.h | 19 +--
- 2 files changed, 14 insertions(+), 7 deletions(-)
-
-diff --git a/agent/tcf/framework/channel_lws.c 
b/agent/tcf/framework/channel_lws.c
-index 0cb9585..d9352f3 100644
 a/tcf/framework/channel_lws.c
-+++ b/tcf/framework/channel_lws.c
-@@ -1,5 +1,5 @@
- 
/***
-- * Copyright (c) 2016-2017 Wind River Systems, Inc.
-+ * Copyright (c) 2016-2018 Wind River Systems, Inc. and others.
-  * All rights reserved. This program and the accompanying materials
-  * are made available under the terms of the Eclipse Public License v1.0
-  * and Eclipse Distribution License v1.0 which accompany this distribution.
-diff --git a/agent/tcf/framework/cpudefs-mdep-mux.h 
b/agent/tcf/framework/cpudefs-mdep-mux.h
-index c9e0db7..0397a6a 100644
 a/tcf/framework/cpudefs-mdep-mux.h
-+++ b/tcf/framework/cpudefs-mdep-mux.h
-@@ -1,10 +1,17 @@
--/*
-- * Copyright (c) 2013 Wind River Systems, Inc.
-+/***
-+ * Copyright (c) 2013 Wind River Systems, Inc. and others.
-+ * All rights reserved. This program and the accompanying materials
-+ * are made available under the terms of the Eclipse Public License v1.0
-+ * and Eclipse Distribution License v1.0 which accompany this distribution.
-+ * The Eclipse Public License is available at
-+ * http://www.eclipse.org/legal/epl-v10.html
-+ * and the Eclipse Distribution License is available at
-+ * http://www.eclipse.org/org/documents/edl-v10.php.
-+ * You may elect to redistribute this code under either of these licenses.
-  *
-- * The right to copy, distribute, modify or otherwise make use
-- * of this software may be licensed only pursuant to the terms
-- * of an applicable Wind River license agreement.
-- */
-+ * Contributors:
-+ * Wind River Systems - initial API and implementation
-+ 
***/
- 
- #include 
- 
--- 
-2.7.4
-
diff --git a/meta/recipes-devtools/tcf-agent/tcf-agent_git.bb 
b/meta/recipes-devtools/tcf-agent/tcf-agent_git.bb
index 4ff309dd80d..e67eccc75ce 100644
--- a/meta/recipes-devtools/tcf-agent/tcf-agent_git.bb
+++ b/meta/recipes-devtools/tcf-agent/tcf-agent_git.bb
@@ -6,7 +6,7 @@ BUGTRACKER = "https://bugs.eclipse.org/bugs/;
 LICENSE = "EPL-1.0 | EDL-1.0"
 LIC_FILES_CHKSUM = "file://edl-v10.html;md5=522a390a83dc186513f0500543ad3679"
 
-SRCREV = "a022ef2f1acfd9209a1bf792dda14ae4b0d1b60f"
+SRCREV = "2735e3d6b7eccb05ab232825c618c837d27a5010"
 PV = "1.7.0+git${SRCPV}"
 
 UPSTREAM_CHECK_GITTAGREGEX = "(?P(\d+(\.\d+)+))"
@@ -15,7 +15,6 @@ SRC_URI = 
"git://git.eclipse.org/r/tcf/org.eclipse.tcf.agent.git;protocol=https
file://ldflags.patch \
file://tcf-agent.init \
file://tcf-agent.service \
-   file://0001-Fixed-copyright-messages.patch \
   "
 
 DEPENDS = "util-linux openssl"
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156119): 
https://lists.openembedded.org/g/openembedded-core/message/156119
Mute This Topic: https://lists.openembedded.org/mt/85657893/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core][PATCH v3 2/2] externalsrc: Work with reproducible_build

[Mark Hatle]

From: Mark Hatle 

Externalsrc removes do_fetch, do_unpack, and do_patch.  The system normally
discovers the correct reproducible date as a postfuncs of do_unpack, so this
date is never found, so it falls back to the default epoch.

Instead we can move the discovery function to a prefuncs on the epoch
deploy task.  This task will run before do_configure, and since the source
is already available can run safely at anytime.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/classes/externalsrc.bbclass | 10 ++
 1 file changed, 10 insertions(+)

diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass
index 54b08adf62..7f1a760eec 100644
--- a/meta/classes/externalsrc.bbclass
+++ b/meta/classes/externalsrc.bbclass
@@ -110,6 +110,16 @@ python () {
 continue
 bb.build.deltask(task, d)
 
+if bb.data.inherits_class('reproducible_build', d) and 'do_unpack' in 
d.getVar("SRCTREECOVEREDTASKS").split():
+# The reproducible_build's create_source_date_epoch_stamp function 
must
+# be run after the source is available and before the
+# do_deploy_source_date_epoch task.  In the normal case, it's 
attached
+# to do_unpack as a postfuncs, but since we removed do_unpack 
(above)
+# we need to move the function elsewhere.  The easiest thing to do 
is
+# move it into the prefuncs of the do_deploy_source_date_epoch 
task.
+# This is safe, as externalsrc runs with the source already 
unpacked.
+d.prependVarFlag('do_deploy_source_date_epoch', 'prefuncs', 
'create_source_date_epoch_stamp ')
+
 d.prependVarFlag('do_compile', 'prefuncs', 
"externalsrc_compile_prefunc ")
 d.prependVarFlag('do_configure', 'prefuncs', 
"externalsrc_configure_prefunc ")
 
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#155881): 
https://lists.openembedded.org/g/openembedded-core/message/155881
Mute This Topic: https://lists.openembedded.org/mt/85500953/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core][PATCH v3 0/2] Fixes for reproducible build and externalsrc

[Mark Hatle]

v3:
SAME AS V2, except I corrected a minor typographical error in a comment in
2/2.

v2:
I've reworked the patches completely based on the feedback and a discusion
with RP on IRC.

The first patch both simplifies and expands the description of what the
class is doing.  (To fully disable the behavior for a single recipe that
recipe would need to delVar SOURCE_DATE_EPOCH.  With this set, all sorts
of programs pay attention to the value, if it's blank it may error,
otherwise it falls back to either 0, or another date.)

The documentation is expanded to make it clear how to override the behavior
but it was not desired to explain how to 'disable' the behavior for a
single recipe.

For the externalsrc, the code was moved from the reproducible_build into
the externalsrc and made contingent on the do_unpack being removed.

Mark Hatle (2):
  reproducible_build: Remove BUILD_REPRODUCIBLE_BINARIES checking
  externalsrc: Work with reproducible_build

 meta/classes/externalsrc.bbclass| 10 +
 meta/classes/reproducible_build.bbclass | 53 -
 2 files changed, 44 insertions(+), 19 deletions(-)

-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#155880): 
https://lists.openembedded.org/g/openembedded-core/message/155880
Mute This Topic: https://lists.openembedded.org/mt/85500952/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core][PATCH v3 1/2] reproducible_build: Remove BUILD_REPRODUCIBLE_BINARIES checking

[Mark Hatle]

From: Mark Hatle 

Previously if BUILD_REPRODUCIBLE_BINARIES was set to 0, the system would
fall back and select the default epoch (April 2011), but still perform
the reproducible build actions.  This resulted in binaries that had an
unusually old date.

Simplify the functions and remove the anonymous python as no longer
necessary.

Also improve the documentation to better explain what the class is doing
and how a recipe can override the behavior if necessary.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/classes/reproducible_build.bbclass | 53 -
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/meta/classes/reproducible_build.bbclass 
b/meta/classes/reproducible_build.bbclass
index 378121903d..96bb012243 100644
--- a/meta/classes/reproducible_build.bbclass
+++ b/meta/classes/reproducible_build.bbclass
@@ -1,17 +1,38 @@
 # reproducible_build.bbclass
 #
-# Sets SOURCE_DATE_EPOCH in each component's build environment.
+# Sets the default SOURCE_DATE_EPOCH in each component's build environment.
+# The format is number of seconds since the system epoch.
+#
 # Upstream components (generally) respect this environment variable,
 # using it in place of the "current" date and time.
 # See https://reproducible-builds.org/specs/source-date-epoch/
 #
-# After sources are unpacked but before they are patched, we set a 
reproducible value for SOURCE_DATE_EPOCH.
-# This value should be reproducible for anyone who builds the same revision 
from the same sources.
+# The default value of SOURCE_DATE_EPOCH comes from the function
+# get_source_date_epoch_value which reads from the SDE_FILE, or if the file
+# is not available (or set to 0) will use the fallback of
+# SOURCE_DATE_EPOCH_FALLBACK.
+#
+# The SDE_FILE is normally constructed from the function
+# create_source_date_epoch_stamp which is typically added as a postfuncs to
+# the do_unpack task.  If a recipe does NOT have do_unpack, it should be added
+# to a task that runs after the source is available and before the
+# do_deploy_source_date_epoch task is executed.
+#
+# If a recipe wishes to override the default behavior it should set it's own
+# SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task
+# with recipe-specific functionality to write the appropriate
+# SOURCE_DATE_EPOCH into the SDE_FILE.
+#
+# SOURCE_DATE_EPOCH is intended to be a reproducible value.  This value should
+# be reproducible for anyone who builds the same revision from the same
+# sources.
 #
-# There are 4 ways we determine SOURCE_DATE_EPOCH:
+# There are 4 ways the create_source_date_epoch_stamp function determines what
+# becomes SOURCE_DATE_EPOCH:
 #
 # 1. Use the value from __source_date_epoch.txt file if this file exists.
-#This file was most likely created in the previous build by one of the 
following methods 2,3,4.
+#This file was most likely created in the previous build by one of the
+#following methods 2,3,4.
 #Alternatively, it can be provided by a recipe via SRC_URI.
 #
 # If the file does not exist:
@@ -22,20 +43,16 @@
 # 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ...
 #This works for well-kept repositories distributed via tarball.
 #
-# 4. Use the modification time of the youngest file in the source tree, if 
there is one.
+# 4. Use the modification time of the youngest file in the source tree, if
+#there is one.
 #This will be the newest file from the distribution tarball, if any.
 #
-# 5. Fall back to a fixed timestamp.
+# 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK).
 #
-# Once the value of SOURCE_DATE_EPOCH is determined, it is stored in the 
recipe's SDE_FILE.
-# If none of these mechanisms are suitable, replace the 
do_deploy_source_date_epoch task
-# with recipe-specific functionality to write the appropriate 
SOURCE_DATE_EPOCH into the SDE_FILE.
-#
-# If this file is found by other tasks, the value is exported in the 
SOURCE_DATE_EPOCH variable.
-# SOURCE_DATE_EPOCH is set for all tasks that might use it (do_configure, 
do_compile, do_package, ...)
+# Once the value is determined, it is stored in the recipe's SDE_FILE.
 
 BUILD_REPRODUCIBLE_BINARIES ??= '1'
-inherit ${@oe.utils.ifelse(d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1', 
'reproducible_build_simple', '')}
+inherit reproducible_build_simple
 
 SDE_DIR = "${WORKDIR}/source-date-epoch"
 SDE_FILE = "${SDE_DIR}/__source_date_epoch.txt"
@@ -92,6 +109,9 @@ python create_source_date_epoch_stamp() {
 os.rename(tmp_file, epochfile)
 }
 
+# Generate the stamp after do_unpack runs
+do_unpack[postfuncs] += "create_source_date_epoch_stamp"
+
 def get_source_date_epoch_value(d):
 cached = d.getVar('__CACHED_SOURCE_DATE_EPOCH')
 if cached:
@@ -120,8 +140,3 @@ def get_source_date_epoch_value(d):
 
 export SOURCE_DATE_EPOCH ?= "${@get_source_date_epoch_value(d)}"
 BB_HASHBASE_WHITELIST += "SOURCE_DATE_EP

[OE-core][PATCH v2 0/2] Fixes for reproducible build and externalsrc

[Mark Hatle]

I've reworked the patches completely based on the feedback and a discusion
with RP on IRC.

The first patch both simplifies and expands the description of what the
class is doing.  (To fully disable the behavior for a single recipe that
recipe would need to delVar SOURCE_DATE_EPOCH.  With this set, all sorts
of programs pay attention to the value, if it's blank it may error,
otherwise it falls back to either 0, or another date.)

The documentation is expanded to make it clear how to override the behavior
but it was not desired to explain how to 'disable' the behavior for a
single recipe.

For the externalsrc, the code was moved from the reproducible_build into
the externalsrc and made contingent on the do_unpack being removed.

Mark Hatle (2):
  reproducible_build: Remove BUILD_REPRODUCIBLE_BINARIES checking
  externalsrc: Work with reproducible_build

 meta/classes/externalsrc.bbclass| 10 +
 meta/classes/reproducible_build.bbclass | 53 -
 2 files changed, 44 insertions(+), 19 deletions(-)

-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#155878): 
https://lists.openembedded.org/g/openembedded-core/message/155878
Mute This Topic: https://lists.openembedded.org/mt/85500124/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core][PATCH v2 1/2] reproducible_build: Remove BUILD_REPRODUCIBLE_BINARIES checking

[Mark Hatle]

From: Mark Hatle 

Previously if BUILD_REPRODUCIBLE_BINARIES was set to 0, the system would
fall back and select the default epoch (April 2011), but still perform
the reproducible build actions.  This resulted in binaries that had an
unusually old date.

Simplify the functions and remove the anonymous python as no longer
necessary.

Also improve the documentation to better explain what the class is doing
and how a recipe can override the behavior if necessary.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/classes/reproducible_build.bbclass | 53 -
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/meta/classes/reproducible_build.bbclass 
b/meta/classes/reproducible_build.bbclass
index 378121903d..96bb012243 100644
--- a/meta/classes/reproducible_build.bbclass
+++ b/meta/classes/reproducible_build.bbclass
@@ -1,17 +1,38 @@
 # reproducible_build.bbclass
 #
-# Sets SOURCE_DATE_EPOCH in each component's build environment.
+# Sets the default SOURCE_DATE_EPOCH in each component's build environment.
+# The format is number of seconds since the system epoch.
+#
 # Upstream components (generally) respect this environment variable,
 # using it in place of the "current" date and time.
 # See https://reproducible-builds.org/specs/source-date-epoch/
 #
-# After sources are unpacked but before they are patched, we set a 
reproducible value for SOURCE_DATE_EPOCH.
-# This value should be reproducible for anyone who builds the same revision 
from the same sources.
+# The default value of SOURCE_DATE_EPOCH comes from the function
+# get_source_date_epoch_value which reads from the SDE_FILE, or if the file
+# is not available (or set to 0) will use the fallback of
+# SOURCE_DATE_EPOCH_FALLBACK.
+#
+# The SDE_FILE is normally constructed from the function
+# create_source_date_epoch_stamp which is typically added as a postfuncs to
+# the do_unpack task.  If a recipe does NOT have do_unpack, it should be added
+# to a task that runs after the source is available and before the
+# do_deploy_source_date_epoch task is executed.
+#
+# If a recipe wishes to override the default behavior it should set it's own
+# SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task
+# with recipe-specific functionality to write the appropriate
+# SOURCE_DATE_EPOCH into the SDE_FILE.
+#
+# SOURCE_DATE_EPOCH is intended to be a reproducible value.  This value should
+# be reproducible for anyone who builds the same revision from the same
+# sources.
 #
-# There are 4 ways we determine SOURCE_DATE_EPOCH:
+# There are 4 ways the create_source_date_epoch_stamp function determines what
+# becomes SOURCE_DATE_EPOCH:
 #
 # 1. Use the value from __source_date_epoch.txt file if this file exists.
-#This file was most likely created in the previous build by one of the 
following methods 2,3,4.
+#This file was most likely created in the previous build by one of the
+#following methods 2,3,4.
 #Alternatively, it can be provided by a recipe via SRC_URI.
 #
 # If the file does not exist:
@@ -22,20 +43,16 @@
 # 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ...
 #This works for well-kept repositories distributed via tarball.
 #
-# 4. Use the modification time of the youngest file in the source tree, if 
there is one.
+# 4. Use the modification time of the youngest file in the source tree, if
+#there is one.
 #This will be the newest file from the distribution tarball, if any.
 #
-# 5. Fall back to a fixed timestamp.
+# 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK).
 #
-# Once the value of SOURCE_DATE_EPOCH is determined, it is stored in the 
recipe's SDE_FILE.
-# If none of these mechanisms are suitable, replace the 
do_deploy_source_date_epoch task
-# with recipe-specific functionality to write the appropriate 
SOURCE_DATE_EPOCH into the SDE_FILE.
-#
-# If this file is found by other tasks, the value is exported in the 
SOURCE_DATE_EPOCH variable.
-# SOURCE_DATE_EPOCH is set for all tasks that might use it (do_configure, 
do_compile, do_package, ...)
+# Once the value is determined, it is stored in the recipe's SDE_FILE.
 
 BUILD_REPRODUCIBLE_BINARIES ??= '1'
-inherit ${@oe.utils.ifelse(d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1', 
'reproducible_build_simple', '')}
+inherit reproducible_build_simple
 
 SDE_DIR = "${WORKDIR}/source-date-epoch"
 SDE_FILE = "${SDE_DIR}/__source_date_epoch.txt"
@@ -92,6 +109,9 @@ python create_source_date_epoch_stamp() {
 os.rename(tmp_file, epochfile)
 }
 
+# Generate the stamp after do_unpack runs
+do_unpack[postfuncs] += "create_source_date_epoch_stamp"
+
 def get_source_date_epoch_value(d):
 cached = d.getVar('__CACHED_SOURCE_DATE_EPOCH')
 if cached:
@@ -120,8 +140,3 @@ def get_source_date_epoch_value(d):
 
 export SOURCE_DATE_EPOCH ?= "${@get_source_date_epoch_value(d)}"
 BB_HASHBASE_WHITELIST += "SOURCE_DATE_EP

[OE-core][PATCH v2 2/2] externalsrc: Work with reproducible_build

[Mark Hatle]

From: Mark Hatle 

Externalsrc removes do_fetch, do_unpack, and do_patch.  The system normally
discovers the correct reproducible date as a postfuncs of do_unpack, so this
date is never found, so it falls back to the default epoch.

Instead we can move the discovery function to a prefuncs on the epoch
deploy task.  This task will run before do_configure, and since the source
is already available can run safely at anytime.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/classes/externalsrc.bbclass | 10 ++
 1 file changed, 10 insertions(+)

diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass
index 54b08adf62..34921b39a2 100644
--- a/meta/classes/externalsrc.bbclass
+++ b/meta/classes/externalsrc.bbclass
@@ -110,6 +110,16 @@ python () {
 continue
 bb.build.deltask(task, d)
 
+if bb.data.inherits_class('reproducible_build', d) and 'do_unpack' in 
d.getVar("SRCTREECOVEREDTASKS").split():
+# The reproducible_build's create_source_date_epoch_stamp function 
must
+# be run after the source is available and before the
+# do_deploy_source_date_epoch task.  In the normal case, it's 
attached
+# to do_unpack as a postfuncs, but since we removed do_unpack 
(above)
+# we need to move the function elsewhere.  The easiest thing to do 
is
+# move it into the prefuncs of the do_deploy_source_date_epoch 
task.
+# This is safe, as externalsrc runs with the source is already 
unpacked.
+d.prependVarFlag('do_deploy_source_date_epoch', 'prefuncs', 
'create_source_date_epoch_stamp ')
+
 d.prependVarFlag('do_compile', 'prefuncs', 
"externalsrc_compile_prefunc ")
 d.prependVarFlag('do_configure', 'prefuncs', 
"externalsrc_configure_prefunc ")
 
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#155876): 
https://lists.openembedded.org/g/openembedded-core/message/155876
Mute This Topic: https://lists.openembedded.org/mt/85500122/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core][PATCH 1/2] reproducible_build: Honor BUILD_REPRODUCIBLE_BINARIES

[Mark Hatle]

From: Mark Hatle 

A variable BUILD_REPRODUCIBLE_BINARIES is set to '1' by default and was used
to control if the postfuncs were added.  However, it didn't actually disable
the reproducible_build (date) logic.  This resulted in the program falling
back to the default date.

This change fully honors the variable and disables the various pieces
that discover and configure the SOURCE_DATE_EPOCH if the value if not
set to '1'.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/classes/reproducible_build.bbclass | 17 ++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/meta/classes/reproducible_build.bbclass 
b/meta/classes/reproducible_build.bbclass
index 378121903d..a9c117c3b9 100644
--- a/meta/classes/reproducible_build.bbclass
+++ b/meta/classes/reproducible_build.bbclass
@@ -47,7 +47,9 @@ TARGET_CC_ARCH:append:class-target = " -Wdate-time"
 # A SOURCE_DATE_EPOCH of '0' might be misinterpreted as no SDE
 export SOURCE_DATE_EPOCH_FALLBACK ??= "1302044400"
 
-SSTATETASKS += "do_deploy_source_date_epoch"
+# The following are preformed in the anonymous python function, only if
+# BUILD_REPRODUCIBLE_BINARIES == 1
+#SSTATETASKS += "do_deploy_source_date_epoch"
 
 do_deploy_source_date_epoch () {
 mkdir -p ${SDE_DEPLOYDIR}
@@ -73,8 +75,10 @@ python do_deploy_source_date_epoch_setscene () {
 
 do_deploy_source_date_epoch[dirs] = "${SDE_DEPLOYDIR}"
 do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}"
-addtask do_deploy_source_date_epoch_setscene
-addtask do_deploy_source_date_epoch before do_configure after do_patch
+# The following are preformed in the anonymous python function, only if
+# BUILD_REPRODUCIBLE_BINARIES == 1
+#addtask do_deploy_source_date_epoch_setscene
+#addtask do_deploy_source_date_epoch before do_configure after do_patch
 
 python create_source_date_epoch_stamp() {
 import oe.reproducible
@@ -123,5 +127,12 @@ BB_HASHBASE_WHITELIST += "SOURCE_DATE_EPOCH"
 
 python () {
 if d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1':
+# Generate the timestamp with create_source_date_epoch_stamp.
 d.appendVarFlag("do_unpack", "postfuncs", " 
create_source_date_epoch_stamp")
+d.appendVar('SSTATETASKS', " do_deploy_source_date_epoch")
+bb.build.addtask('do_deploy_source_date_epoch_setscene', None, None, d)
+bb.build.addtask('do_deploy_source_date_epoch', 'do_configure', 
'do_patch', d)
+else:
+# If this is set at all, the system components will attempt to use it
+d.delVar('SOURCE_DATE_EPOCH')
 }
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#155839): 
https://lists.openembedded.org/g/openembedded-core/message/155839
Mute This Topic: https://lists.openembedded.org/mt/85476713/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core][PATCH 0/2] Fixes for reproducible build and externalsrc

[Mark Hatle]

Two fixes, the first one ensures that the BUILD_REPRODUCIBLE_BINARIES variable
actually does something useful.  1 (default) normal behavior, anything else it
is off and regular behavior works.

The second makes the reproducible build work properly with externalsrc.

To test this, I did:

bitbake bash
rpm -qp tmp/deploy/.../bash-...rpm --queryformat '[%{FILENAMES} 
%{FILEMTIMES}\n]'

cleanall the above, and then add:

BUILD_REPRODUCIBLE_BINARIES:pn-bash = '0'

repeat the test...

You can do the same with externalsrc via a bbappen adding externalsrc and repeat
the previous tests and get expected results.

Mark Hatle (2):
  reproducible_build: Honor BUILD_REPRODUCIBLE_BINARIES
  reproducible_build: Work with externalsrc

 meta/classes/reproducible_build.bbclass | 25 +
 1 file changed, 21 insertions(+), 4 deletions(-)

-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#155838): 
https://lists.openembedded.org/g/openembedded-core/message/155838
Mute This Topic: https://lists.openembedded.org/mt/85476712/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core][PATCH 2/2] reproducible_build: Work with externalsrc

[Mark Hatle]

From: Mark Hatle 

Externalsrc removes do_fetch, do_unpack, and do_patch.  The system normally
discovers the correct reproducible date as a postfuncs on do_unpack, so this
date is never found, so it goes back to the default epoch.

Instead we can move the discovery function to a prefuncs on the epoch
deploy task.  This task will run before do_configure, and since the source
is already available can run anytime safely.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/classes/reproducible_build.bbclass | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/meta/classes/reproducible_build.bbclass 
b/meta/classes/reproducible_build.bbclass
index a9c117c3b9..ae0723ab21 100644
--- a/meta/classes/reproducible_build.bbclass
+++ b/meta/classes/reproducible_build.bbclass
@@ -128,7 +128,13 @@ BB_HASHBASE_WHITELIST += "SOURCE_DATE_EPOCH"
 python () {
 if d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1':
 # Generate the timestamp with create_source_date_epoch_stamp.
-d.appendVarFlag("do_unpack", "postfuncs", " 
create_source_date_epoch_stamp")
+# In most cases this will be a postfuncs of do_unpack.
+# If we're running in external source, add 
create_source_date_epoch_stamp
+# to do_deploy_source_date_epoch instead because there is no do_unpack.
+if d.getVar('EXTERNALSRC') and bb.data.inherits_class('externalsrc', 
d):
+d.appendVarFlag('do_deploy_source_date_epoch', 'prefuncs', ' 
create_source_date_epoch_stamp')
+else:
+d.appendVarFlag("do_unpack", "postfuncs", " 
create_source_date_epoch_stamp")
 d.appendVar('SSTATETASKS', " do_deploy_source_date_epoch")
 bb.build.addtask('do_deploy_source_date_epoch_setscene', None, None, d)
 bb.build.addtask('do_deploy_source_date_epoch', 'do_configure', 
'do_patch', d)
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#155840): 
https://lists.openembedded.org/g/openembedded-core/message/155840
Mute This Topic: https://lists.openembedded.org/mt/85476714/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] should recipes not come with their own systemd/user/group info?

[Mark Hatle]

On 8/5/21 6:01 AM, Robert P. J. Day wrote:
> 
>   style question here, but i'm perusing an OE/YP (actually wind river,
> but effectively the same thing) layer that was handed to me, and i'm
> puzzled by how the internally-written systemd-controlled services are
> implemented.
> 
>   on the one hand, there are reasonable recipes that each represent
> some service to be managed by systemd, and they all install their
> executables in the right places and so on. however, rather than each
> recipe also providing its own service file, there is a totally
> separate recipe (call it "systemd-services.bb") which contains under
> its files/ directory dozens and dozens of service files for all of
> those recipes, and is responsible for installing all of them.
> 
>   in short, someone decided that -- for whatever reason -- all of the
> systemd service files should be collected all in one place, and
> installed en masse. is this ... normal? i would have thought that it's
> the responsibility of each recipe to mannage all the content related
> to it, and that would of course include the systemd service file to be
> installed for it. i don't know the rationale for this, and it strikes
> me as a recipe for disaster in trying to keep all these things synced
> up. is there some rationale for doing it this way that i'm unaware
> of?

For a 'product' (as in, this is only for one implementation and nobody will ever
re-use it), I've see this type of thing before.  But for anything intended to be
re-used, it's simply broken.

The service files belong with the recipes providing the services.

>   and closely related, the same trick was used when defining all the
> new users and groups that would own these services. rather than each
> service looking after itself, there is a recipe file (call it
> "new_users_and_groups.bb") which is hundreds of lines of USERADD_PARAM
> and GROUPADD_PARAM lines that define precisely (numerically) all the
> new user and group accounts to be associated with all sorts of
> services, whereupon i ask the same question -- isn't it more typical
> that each service look after its own user and group info?

The same is true here, if you are creating a singular product that has a number
of defined users and groups, sure you can do it in a recipe like this.  But it's
not the right way to do it.  Users/groups should be associated with their users.
 If these are just general administrative usages then they COULD be done this
way, but more people do it at image generation time.

>   in this second case, it *might* be that all of these values need to
> be hard-coded to be consistent with, i don't know, something else i'm
> unaware of, but it still looks strange.

If you need to 'adjust' users and groups, you should be using the existing
mechanisms, useradd-staticids for instance.

--Mark

>   thoughts? the above actually works for the time being, but it does
> seem a but unnatural.
> 
> rday
> 
> 
> 
> 
> 

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#154522): 
https://lists.openembedded.org/g/openembedded-core/message/154522
Mute This Topic: https://lists.openembedded.org/mt/84682032/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH 6/9] shadow: update 4.8.1 -> 4.9

[Mark Hatle]

On 8/4/21 1:13 PM, Khem Raj wrote:
> 
> 
> On 8/4/21 3:12 AM, Alexander Kanavin wrote:
>> Yes, plaintext passwords can no longer be there, which is a good thing 
>> I'd say? The hashed/salted passwords can still be provided through the 
>> same class, but this needs to be documented, and perhaps tested too.
>>
> 
> Its perhaps fine to discourage plaintext password setting, but it is a 
> user visible feature as it seems. So the documentation should change for 
> sure to not use it and it should also go into migration guide since it 
> has a potential of tripping a lot of folks. I think documenting the 
> intent to move away from plaintext is urgent, then the question is if
> we want to fist deprecate it or delete this option all in one go.

We SHOULD discourage users from any hardcoded passwords!  But, there is little
to no functional difference between specifying a plain text or salted password,
but there is a HUGE developer/user difference in behavior.

So, if we have a way to set a default password for any account, then we really
do need a way to have a plaintext password specified.

>From a security perspective, there is no advantage between a salted or plain
text password.  (Salted passwords can always be reversed through tables, etc!)

If the current implementation of the plain text passwords is not "secure" due to
bad salts, hash types, etc.  Then lets fix that and move to a more secure style.

If it is decided to remove the -P option for plain text passwords, then we need
to document for the user HOW to generate password hashes.  And if we're showing
them how to do it, it SHOULD be trivial to find a way to do the same thing
_using the build system_.

For example

useradd -P 'foobar' user

to

hash=$(echo 'foobar' | openssl passwd -1 -salt mysalt -stdin)
useradd -p $hash user


or

hash=$(python -c "import crypt; print crypt.crypt('foobar')")
useradd -p $hash user


or




but the point is, we SHOULD discourage _ANY_ hard coded passwords, not just
plain text.  However if a user wants to do this, the system should assist the
user in setting a password into their environment.

--Mark


>> Alex
>>
>> On Wed, 4 Aug 2021 at 10:39, Yi Zhao > > wrote:
>>
>>
>> On 7/30/21 7:45 PM, Alexander Kanavin wrote:
>>> Add a couple backports to fix builds.
>>>
>>> Drop 0002-Allow-for-setting-password-in-clear-text.patch;
>>> what it adds is horribly insecure and AB testing didn't reveal any
>>> regressions or use cases for it.
>>
>> Dropping this patch makes the password setting function in
>> extrausers.bbclass unavailable:
>> https://docs.yoctoproject.org/singleindex.html#extrausers-bbclass
>> 
>>
>>
>> //Yi
>>
>>
>>> Drop /etc/default/ tweaks as files are no longer installed there.
>>>
>>> Drop manpage alternatives as manpages are no longer installed.
>>>
>>> Signed-off-by: Alexander Kanavin  
>>> 
>>> ---
>>>   ...01-Disable-use-of-syslog-for-sysroot.patch |  29 +-
>>>   ...builds-with-respect-to-libsubid-incl.patch | 114 +++
>>>   .../0001-libsubid-link-to-PAM-libraries.patch |  31 ++
>>>   ...w-for-setting-password-in-clear-text.patch | 301 --
>>>   ...nexpected-open-failure-in-chroot-env.patch |   6 +-
>>>   meta/recipes-extended/shadow/shadow.inc   |  21 +-
>>>   .../shadow/{shadow_4.8.1.bb    
>>> =>shadow_4.9.bb  } |   0
>>>   7 files changed, 167 insertions(+), 335 deletions(-)
>>>   create mode 100644 
>>> meta/recipes-extended/shadow/files/0001-Fix-out-of-tree-builds-with-respect-to-libsubid-incl.patch
>>>   create mode 100644 
>>> meta/recipes-extended/shadow/files/0001-libsubid-link-to-PAM-libraries.patch
>>>   delete mode 100644 
>>> meta/recipes-extended/shadow/files/0002-Allow-for-setting-password-in-clear-text.patch
>>>   rename meta/recipes-extended/shadow/{shadow_4.8.1.bb  
>>>   =>shadow_4.9.bb  } (100%)
>>>
>>> diff --git 
>>> a/meta/recipes-extended/shadow/files/0001-Disable-use-of-syslog-for-sysroot.patch
>>>  
>>> b/meta/recipes-extended/shadow/files/0001-Disable-use-of-syslog-for-sysroot.patch
>>> index ab317b9aa0..95728bcd3f 100644
>>> --- 
>>> a/meta/recipes-extended/shadow/files/0001-Disable-use-of-syslog-for-sysroot.patch
>>> +++ 
>>> b/meta/recipes-extended/shadow/files/0001-Disable-use-of-syslog-for-sysroot.patch
>>> @@ -1,4 +1,4 @@
>>> -From fa2d9453656641002802d8165e80adb9e6a729d2 Mon Sep 17 00:00:00 2001
>>> +From 30a3906a0a21120fa6bbc918b6258ab9303fbeaa Mon Sep 17 00:00:00 2001
>>>   From: Scott Garman  
>>> 
>>>   Date: Thu, 14 Apr 2016 12:28:57 +0200
>>>   Subject: [PATCH] Disable use of syslog for sysroot
>>> @@ -19,12 +19,12 @@ Signed-off-by: Chen Qi  

Re: [OE-core] Image creation using RPM/DNF and adding a third party repository configuration

[Mark Hatle]

On 7/15/21 4:24 PM, Mark Hatle wrote:
> We have the desire to add a 3rd party repository to our standard images
> generated by OE image generation.  We do this by adding a recipe
> (my-external-repo.bb) that creates a package that adds a repository
> configuration file to ${sysconfdir}/yum.repos.d/my_external.repo.
> 
> If the package is added to the INSTALL_IMAGE, then when it processes the
> IMAGE_FEATURES the system will fail with SSL errors trying to access the new
> third party repository.
> 
> What I believe is happening is that the system (first pass) installs all of 
> the
> main packages, including this repository configuration.  It then starts a 
> second
> pass to install -dev, -src or other components.  This second pass fails with 
> the
> SSL errors (like DNF can't access the SSL certification(s) it needs.)  But 
> even
> if the SSL issue wasn't really a problem, it does point out that DNF is trying
> to access the network and could install something from this third party
> repository, which I don't think is desired.
> 
> An alternative could be to add the third party repository via the ROOTFS POST
> install actions, but this has the problem that we won't be able to update the
> repository if something changes (via a package).
> 
> I'm thinking this MIGHT be a bug in the current implementation, that if 
> someone
> injects a config file it can cause the system to use alternative repositories.
> So should we be checking the directory between passes and sanitizing it?
> 
> Looking for suggestions...
> 
> Thanks!
> --Mark
> 

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#153896): 
https://lists.openembedded.org/g/openembedded-core/message/153896
Mute This Topic: https://lists.openembedded.org/mt/84235991/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH] populate_sdk_ext: Error if trying to generate an eSDK from a mulitconfig

[Mark Hatle]

Running a build such as:

  bitbake mc:my_config:core-image-minimal -c populate_sdk-ext

will result in an error like:

ERROR: Task base-files.do_fetch attempted to execute unexpectedly
Task 
.../poky/build-mc/tmp-my_config-glibc/work/qemuarm64-poky-linux/core-image-minimal/1.0-r0/sdk-ext/image/tmp-renamed-sdk/layers/poky/meta/recipes-core/base-files/base-files_3.0.14.bb:do_packagedata,
 unihash ec5ba0e6b31561daba005fb49c5239c8e46913465b51166b5905f3e5ffcf2741, 
taskhash ec5ba0e6b31561daba005fb49c5239c8e46913465b51166b5905f3e5ffcf2741
Task 
.../poky/build-mc/tmp-my_config-glibc/work/qemuarm64-poky-linux/core-image-minimal/1.0-r0/sdk-ext/image/tmp-renamed-sdk/layers/poky/meta/recipes-core/base-files/base-files_3.0.14.bb:do_package_write_rpm,
 unihash 1c7d7509c2ff6dcf11009fbec444726826214795d60474ec8d3262d89c40a955, 
taskhash 1c7d7509c2ff6dcf11009fbec444726826214795d60474ec8d3262d89c40a955
Task 
.../poky/build-mc/tmp-my_config-glibc/work/qemuarm64-poky-linux/core-image-minimal/1.0-r0/sdk-ext/image/tmp-renamed-sdk/layers/poky/meta/recipes-core/base-files/base-files_3.0.14.bb:do_populate_sysroot,
 unihash 9cc3672f4fa62491f545b15cf617a64cd77d15a2cfd432b57d4b936bc415f40d, 
taskhash 9cc3672f4fa62491f545b15cf617a64cd77d15a2cfd432b57d4b936bc415f40d
Task 
.../poky/build-mc/tmp-my_config-glibc/work/qemuarm64-poky-linux/core-image-minimal/1.0-r0/sdk-ext/image/tmp-renamed-sdk/layers/poky/meta/recipes-core/base-files/base-files_3.0.14.bb:do_package_qa,
 unihash 8ada5f62092c971df8dda1d71c728e42994e1dcf2bbdab419de43867d77b64cc, 
taskhash 8ada5f62092c971df8dda1d71c728e42994e1dcf2bbdab419de43867d77b64cc
Task 
.../poky/build-mc/tmp-my_config-glibc/work/qemuarm64-poky-linux/core-image-minimal/1.0-r0/sdk-ext/image/tmp-renamed-sdk/layers/poky/meta/recipes-core/images/core-image-minimal.bb:do_image_qa,
 unihash 16656a339389e407a5fdca5d64983af845288f3b3cc5582398e5247efb393257, 
taskhash 16656a339389e407a5fdca5d64983af845288f3b3cc5582398e5247efb393257
Task 
.../poky/build-mc/tmp-my_config-glibc/work/qemuarm64-poky-linux/core-image-minimal/1.0-r0/sdk-ext/image/tmp-renamed-sdk/layers/poky/meta/recipes-core/images/core-image-minimal.bb:do_image_complete,
 unihash ef88c74a9f4ae4d252c421eb4e399773aa50cea7c51ffbeed9011e5198a16abb, 
taskhash ef88c74a9f4ae4d252c421eb4e399773aa50cea7c51ffbeed9011e5198a16abb
This is usually due to missing setscene tasks. Those missing in this build 
were: 
{'.../poky/build-mc/tmp-my_config-glibc/work/qemuarm64-poky-linux/core-image-minimal/1.0-r0/sdk-ext/image/tmp-renamed-sdk/layers/poky/meta/recipes-core/base-files/base-files_3.0.14.bb:do_package_qa',

Instead of letting the system error, we simply tell the user this is not 
supported.

As long as the eSDK is constructed based on the primary library, it works fine.

Signed-off-by: Mark Hatle 
---
 meta/classes/populate_sdk_ext.bbclass | 5 +
 1 file changed, 5 insertions(+)

diff --git a/meta/classes/populate_sdk_ext.bbclass 
b/meta/classes/populate_sdk_ext.bbclass
index 517b4e45ff0..4aabafa079b 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -750,6 +750,11 @@ fakeroot python do_populate_sdk_ext() {
 if d.getVar('SDK_ARCH') != d.getVar('BUILD_ARCH'):
 bb.fatal('The extensible SDK can currently only be built for the same 
architecture as the machine being built on - SDK_ARCH is set to %s (likely via 
setting SDKMACHINE) which is different from the architecture of the build 
machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH'), 
d.getVar('BUILD_ARCH')))
 
+# FIXME hopefully we can remove this restriction at some point, but the 
eSDK
+# can only be built for the primary (default) multiconfig
+if d.getVar('BB_CURRENT_MC') != 'default':
+bb.fatal('The extensible SDK can currently only be built for the 
default multiconfig.  Currently trying to build for %s.' % 
d.getVar('BB_CURRENT_MC'))
+
 d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d))
 if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1':
 buildtools_fn = get_current_buildtools(d)
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#153434): 
https://lists.openembedded.org/g/openembedded-core/message/153434
Mute This Topic: https://lists.openembedded.org/mt/83876060/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] should the same recipe have two different WORKDIRs?

[Mark Hatle]

Part of the WORKDIR component is the target (package) architecture.  If the
recipe or one of it's inherits sets PACKAGE_ARCH = "${MACHINE_ARCH}" it will
move the component.

Where I've seen both directories used is when someone is building for multiple
target boards.. and some of the targets use the default PACKAGE_ARCH, and some
use MACHINE_ARCH.  This is very common with OpenGL and hardware acceleration.
Mesa for instance becomes machine dependent on some systems, but generic for 
others.

--Mark

On 6/16/21 11:30 AM, Robert P. J. Day wrote:
> 
>   perhaps i've just never noticed before, but a colleague asked me to
> debug some strangeness with his WRLinux build, and what i noticed was
> that the recipe that generated a package with a single (aarch64)
> executable created WORKDIRs under both directories:
> 
>   * cortexa53...
>   * acme-coyote   [actual target board]
> 
> this surprises me ... i thought that, based on the attributes of any
> recipe, it would have only one WORKDIR in the appropriate place. what
> does the above mean? i'm not sure what to make of it.
> 
> rday
> 
> 
> 
> 
> 

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#153028): 
https://lists.openembedded.org/g/openembedded-core/message/153028
Mute This Topic: https://lists.openembedded.org/mt/83584601/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] mklibs-native: Fix build with gcc 11

[Mark Hatle]

On 5/17/21 2:46 PM, Andre McCurdy wrote:
> On Mon, May 17, 2021 at 10:05 AM Jacob Kroon  wrote:
>>
>> In gcc 11 the default mode for C++ is now -std=gnu++17 instead of 
>> -std=gnu++14,
>> in which support for dynamic exception specifications has been removed.
> 
> As much as I'd like to see mklibs fully supported in OE, at some point
> maybe we should just admit that it doesn't work and there's not enough
> collective motivation to fix it.
> 
> For mklibs to work, the build process should generate a PIC .a archive
> for every .so shared library (ie static libs need to be built and
> somehow forced to contain PIC object code - which may mean patching
> Makefiles etc) and then the PIC .a files need to be available
> alongside the .so libs when mklibs runs. We don't have any support for
> doing that, so why keep pretending to support mklibs?

We used to have exactly that.  The fact nobody is complaining it doesn't work
tells me nobody is using this and we should remove it.

The need for this on modern (embedded) systems is significantly less with a
function altnerative to glibc.  (musl)

--Mark

>> See http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0003r5.html
>>
>> Signed-off-by: Jacob Kroon 
>> ---
>>  .../no-dynamic-exception-specifications.patch | 420 ++
>>  .../mklibs/mklibs-native_0.1.44.bb|   1 +
>>  2 files changed, 421 insertions(+)
>>  create mode 100644 
>> meta/recipes-devtools/mklibs/files/no-dynamic-exception-specifications.patch
>>
>> diff --git 
>> a/meta/recipes-devtools/mklibs/files/no-dynamic-exception-specifications.patch
>>  
>> b/meta/recipes-devtools/mklibs/files/no-dynamic-exception-specifications.patch
>> new file mode 100644
>> index 00..5989a67c4f
>> --- /dev/null
>> +++ 
>> b/meta/recipes-devtools/mklibs/files/no-dynamic-exception-specifications.patch
>> @@ -0,0 +1,420 @@
>> +In gcc 11 the default mode for C++ is now -std=gnu++17 instead of 
>> -std=gnu++14,
>> +in which support for dynamic exception specifications has been removed.
>> +
>> +See http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0003r5.html
>> +
>> +Upstream-Status: Pending
>> +
>> +Signed-off-by: Jacob Kroon 
>> +
>> +Index: mklibs-0.1.44/src/mklibs-readelf/elf.cpp
>> +===
>> +--- mklibs-0.1.44.orig/src/mklibs-readelf/elf.cpp
>>  mklibs-0.1.44/src/mklibs-readelf/elf.cpp
>> +@@ -36,7 +36,7 @@ file::~file () throw ()
>> + delete *it;
>> + }
>> +
>> +-file *file::open (const char *filename) throw (std::bad_alloc, 
>> std::runtime_error)
>> ++file *file::open (const char *filename)
>> + {
>> +   struct stat buf;
>> +   int fd;
>> +@@ -72,7 +72,7 @@ file *file::open (const char *filename)
>> + }
>> +
>> + template
>> +-file *file::open_class(uint8_t *mem, size_t len) throw (std::bad_alloc, 
>> std::runtime_error)
>> ++file *file::open_class(uint8_t *mem, size_t len)
>> + {
>> +   switch (mem[EI_DATA])
>> +   {
>> +@@ -86,7 +86,7 @@ file *file::open_class(uint8_t *mem, siz
>> + }
>> +
>> + template 
>> +-file_data<_class, _data>::file_data(uint8_t *mem, size_t len) throw 
>> (std::bad_alloc, std::runtime_error)
>> ++file_data<_class, _data>::file_data(uint8_t *mem, size_t len)
>> + : file(mem, len)
>> + {
>> +   if (mem[EI_CLASS] != _class::id)
>> +@@ -190,7 +190,7 @@ section_data<_class, _data>::section_dat
>> + }
>> +
>> + template 
>> +-void section_data<_class, _data>::update(const file ) throw 
>> (std::bad_alloc)
>> ++void section_data<_class, _data>::update(const file )
>> + {
>> +   const section_type  =
>> + dynamic_cast 
>> &>(file.get_section(file.get_shstrndx()));
>> +@@ -204,7 +204,7 @@ section_type::~sec
>> + }
>> +
>> + template 
>> +-section_real<_class, _data, section_type_DYNAMIC>::section_real(Shdr 
>> *header, uint8_t *mem) throw (std::bad_alloc)
>> ++section_real<_class, _data, section_type_DYNAMIC>::section_real(Shdr 
>> *header, uint8_t *mem)
>> + : section_data<_class, _data>(header, mem)
>> + {
>> +   if (this->type != SHT_DYNAMIC)
>> +@@ -221,7 +221,7 @@ section_real<_class, _data, section_type
>> + }
>> +
>> + template 
>> +-void section_real<_class, _data, section_type_DYNAMIC>::update(const file 
>> ) throw (std::bad_alloc)
>> ++void section_real<_class, _data, section_type_DYNAMIC>::update(const file 
>> )
>> + {
>> +   section_data<_class, _data>::update(file);
>> +
>> +@@ -243,7 +243,7 @@ section_type::~sect
>> + }
>> +
>> + template 
>> +-section_real<_class, _data, section_type_DYNSYM>::section_real(Shdr 
>> *header, uint8_t *mem) throw (std::bad_alloc)
>> ++section_real<_class, _data, section_type_DYNSYM>::section_real(Shdr 
>> *header, uint8_t *mem)
>> + : section_data<_class, _data>(header, mem)
>> + {
>> +   if (this->type != SHT_DYNSYM)
>> +@@ -260,7 +260,7 @@ section_real<_class, _data, section_type
>> + }
>> +
>> + template 
>> +-void section_real<_class, _data, section_type_DYNSYM>::update(const file 
>> ) throw (std::bad_alloc)
>> ++void 

[OE-core] Multiconfig - sstate-cache re-use and error

[Mark Hatle]

I'm cross posting this cause I suspect it's a bitbake bug, but it requires
sstate-cache / setscene stuff which is from the oe-core side of things.  I
really don't know what is causing the problem, but I can at least reproduce it,
even if it ends up taking a while.

The configuration.  I'm building a system that has 4 multiconfigs (plus the
regular configuration), with the output being an SDK (main config), and 4
different rootfs from the multiconfig.  (See below for actual setup details for
the reproducer.)

What is happening.  The system starts building and appears to correctly defer
the building of the duplicate components.  Eventually it will start building the
item that is not deferred, and soon after it appears to change what is deferred
and will attempt to call setscene tasks, but the sstate-cache has not yet been
written to by the first task.  This is occurring on a completely new build, no
existing sstate-cache.

The console-log can show the order in which things are happening for a small
subset, so might be helpful.  In this chunk socat ended up showing this 
behavior:

NOTE: Deferring mc:qemuarm64-one:socat.bb:do_deploy_source_date_epoch after
mc:qemuarm64-four:socat.bb:do_deploy_source_date_epoch
[repeat for do_package]
[repeat for do_package_qa]
[repeat for do_package_write_rpm]
[repeat for do_packagedata]
[repeat for do_populate_lic]
[repeat for do_populate_sysroot]
*** repeat the sequence above for mc:qemuarm64-three which defers for four ***
*** repeat the sequence above for mc:qemuarm64-two which defers for four ***
NOTE: Running task 1362 of 15106 (mc:qemuarm64-four:socat_1.7.4.1.bb:do_fetch)
NOTE: recipe socat-1.7.4.1-r0: task do_fetch: Started
NOTE: recipe socat-1.7.4.1-r0: task do_fetch: Succeeded
NOTE: Running task 1397 of 15106 (mc:qemuarm64-four:socat_1.7.4.1.bb:do_unpack)
NOTE: recipe socat-1.7.4.1-r0: task do_unpack: Started
NOTE: recipe socat-1.7.4.1-r0: task do_unpack: Succeeded
NOTE: Running task 2617 of 15106 (mc:qemuarm64-four:socat_1.7.4.1.bb:do_patch)
NOTE: recipe socat-1.7.4.1-r0: task do_patch: Started
NOTE: recipe socat-1.7.4.1-r0: task do_patch: Succeeded
NOTE: Running task 2711 of 15106
(mc:qemuarm64-four:socat_1.7.4.1.bb:do_deploy_source_date_epoch)
NOTE: recipe socat-1.7.4.1-r0: task do_deploy_source_date_epoch: Started
NOTE: recipe socat-1.7.4.1-r0: task do_deploy_source_date_epoch: Succeeded
(starts deferring three again after one)
NOTE: Deferring mc:qemuarm64-three:socat_1.7.4.1.bb:do_deploy_source_date_epoch
after mc:qemuarm64-one:socat_1.7.4.1.bb:do_deploy_source_date_epoch
[repeat for do_package]
[repeat for do_package_qa]
[repeat for do_package_write_rpm]
[repeat for do_packagedata]
[repeat for do_populate_lic]
[repeat for do_populate_sysroot]
*** repeat the defer sequence above for mc:qemuarm64-two for one ***
NOTE: Running setscene task 1930 of 5424
(mc:qemuarm64-one:socat_1.7.4.1.bb:do_populate_sysroot_setscene)
NOTE: recipe socat-1.7.4.1-r0: task do_populate_sysroot_setscene: Started
ERROR: mc:qemuarm64-one:socat-1.7.4.1-r0 do_populate_sysroot_setscene: No
suitable staging package found
ERROR: Logfile of failure stored in:
...socat/1.7.4.1-r0/temp/log.do_populate_sysroot_setscene.154008
NOTE: recipe socat-1.7.4.1-r0: task do_populate_sysroot_setscene: Failed
WARNING: Setscene task
(mc:qemuarm64-one:socat_1.7.4.1.bb:do_populate_sysroot_setscene) failed with
exit code '1' - real task will be run instead
NOTE: Running task 3183 of 15106
(mc:qemuarm64-four:/socat_1.7.4.1.bb:do_populate_lic)
... now we have components of 'one' and 'four' building ...  eventually two and
three unlock as well and may build at the same time...

(Note, this build happened to be socat first, but other builds I've run have
different things that end up going first -- and it's not always the same items.)

Reproducer:

latest version of poky

$ . ./oe-init-build-env build-mc
$ mkdir conf/multiconfig

add the following files to multiconfig directory:

conf/multiconfig/qemuarm64-one.conf:
MACHINE = "qemuarm64"
SOC_VARIANT = "one"
TMPDIR = "${TOPDIR}/tmp-${MACHINE}-${SOC_VARIANT}-${TCLIBC}"

conf/multiconfig/qemuarm64-two.conf:
MACHINE = "qemuarm64"
SOC_VARIANT = "two"
TMPDIR = "${TOPDIR}/tmp-${MACHINE}-${SOC_VARIANT}-${TCLIBC}"

conf/multiconfig/qemuarm64-three.conf:
MACHINE = "qemuarm64"
SOC_VARIANT = "three"
TMPDIR = "${TOPDIR}/tmp-${MACHINE}-${SOC_VARIANT}-${TCLIBC}"

conf/multiconfig/qemuarm64-four.conf:
MACHINE = "qemuarm64"
SOC_VARIANT = "four"
TMPDIR = "${TOPDIR}/tmp-${MACHINE}-${SOC_VARIANT}-${TCLIBC}"

edit the conf/local.conf and add:

BBMULTICONFIG += "qemuarm64-one"
BBMULTICONFIG += "qemuarm64-two"
BBMULTICONFIG += "qemuarm64-three"
BBMULTICONFIG += "qemuarm64-four"

Start the build:
$ bitbake buildtools-tarball mc:qemuarm64-one:core-image-minimal
mc:qemuarm64-two:core-image-minimal mc:qemuarm64-three:core-image-minimal
mc:qemuarm64-four:core-image-minimal

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#151347): 

Re: [OE-core] master eSDK hash problem with METADATA_REVISION and base-files:do_install

[Mark Hatle]

I was using master-next (version was right before I sent the email), and it
still failed.

Looking at things, there doesn't appear to be any METADATA_REVISION set in the
normal build, but there is one set inside of the eSDK.  So it looks like some
corner case may have been missing.

If you notice it says .  Diffsigs I had, I don't know how the 'default'
version should be generated or not -- but it was definitely different inside and
outside the eSDK.

--Mark

On 3/28/21 9:11 PM, Chen Qi wrote:
> Hi Mark,
> 
> I think this has been fixed by the following commit.
> 
> """
> commit ff2ad51b801fd62e2abbc573ba2c9ee8fdc7e012
> Author: Chen Qi 
> Date:   Fri Mar 5 18:10:40 2021 +0800
> 
>     populate_sdk_ext: record METADATA_REVISION
>    
>     As we delete the .git/ directory, it's impossible to get METADATA_REVISION
>     inside eSDK. Because of this, we meet the following warning when 
> installing
> eSDK.
>    
>   WARNING: The base-files:do_install sig is computed to be
> 16b9d96148d45de183cc94667aae016ec7d102d48255456381e718cd4bbd0aa0, \
>   but the sig is locked to
> 6eb0dcaed504282becee94662481d79264db920dee1f7deda18230133fff8f36 in
> SIGGEN_LOCKEDSIGS_t-qemux86-64
>    
>     So we record METADATA_REVISION in eSDK generation time to fix this 
> problem.
>    
>     Signed-off-by: Chen Qi 
>     Signed-off-by: Richard Purdie 
> """
> 
> Best Regards,
> Chen Qi
> 
> On 03/26/2021 07:51 AM, Mark Hatle wrote:
>> Building an eSDK (all stock config):
>>
>> MACHINE=qemuarm bitbake core-image-minimal -c populate_sdk_ext
>>
>>
>> I then install it, and get:
>>
>> Checking sstate mirror object availability: 100%
>> |#|
>> Time: 0:00:00
>> WARNING: The base-files:do_install sig is computed to be
>> 587ccfa9299e26094ae342f1ce3d782bfaf776c639013428a511770c3828d49a, but the 
>> sig is
>> locked to 0c35f266f1e79b1f34260df1b8d677f7eb74a7f7241155601ed73a014c8e286c in
>> SIGGEN_LOCKEDSIGS_t-qemuarm
>> Loading cache: 100%
>> |##|
>> Time: 0:00:00
>> Initialising tasks: 100%
>> |#|
>> Time: 0:00:00
>>
>>
>> Running a bitbake diffsigs on this file, I get:
>>
>> basehash changed from
>> cffec52acb67f62fe3812d0d42d52b1081f9150386f43f0426f5469b4a1a6a25 to
>> 332259212c5e9a4ed2a7fe25b8ea77cb54fb6b82f7deed90179e145f1d0cf1b6
>> Variable METADATA_REVISION value changed from
>> '3210cabe56d6cf6d51638e6f31f93110e40ab2cd' to ''
>>
>>
>> So it appears that some of the recent work on METADATA_REVISION has broken 
>> hash
>> computations in the base-files package due to the value being 'unknown' in 
>> one,
>> and set in the other.
>>
>> --Mark
>>
>>
> 
> 
> 
> 
> 

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#150060): 
https://lists.openembedded.org/g/openembedded-core/message/150060
Mute This Topic: https://lists.openembedded.org/mt/81616842/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] populate_sdk_ext: Avoid copying and producing .pyc files

[Mark Hatle]

On 3/26/21 12:54 AM, Alexander Kanavin wrote:
> Wait, I have to say neither reason is convincing. Pyc files are not
> system-specific, they are the same on all systems. And pseudo issues should be
> dealt with by fixing the issue, not working around the symptoms, no?

These are pyc files that are generated by the host system python.  As such, they
contain versioning information of the particular host they were constructed on.

The next user (of the eSDK) would regenerate the pyc files, assuming their
python was a different version/configuration anyway.

The pseudo issue is simply caused by the generation of these files outside of
the pseudo context.  The three places modified need to run without pseudo
control in order to setup the bitbake (host) environment properly, as well as
verify that the previous files copied/setup are infact correct.

So we're not working around symptoms.. we are indeed fixing the underlying
issues in the implementation.  It just happened to be that I was able to
reproduce the build failure reliably for a time vs it just going away on it's 
own.

--Mark

> Alex
> 
> On Thu, 25 Mar 2021 at 23:44, Mark Hatle  <mailto:mark.ha...@kernel.crashing.org>> wrote:
> 
> Since pyc cache files are really system specific, no real reason to copy 
> or
> generate them during the eSDK build process.  Also generating them has the
> possibility of re-using inodes that pseudo may have been tracking, leading
> a build failure.
> 
> Signed-off-by: Mark Hatle  <mailto:mark.ha...@kernel.crashing.org>>
> ---
>  meta/classes/populate_sdk_ext.bbclass | 4 +++-
>  meta/lib/oe/copy_buildsystem.py       | 6 +++---
>  2 files changed, 6 insertions(+), 4 deletions(-)
> 
> diff --git a/meta/classes/populate_sdk_ext.bbclass
> b/meta/classes/populate_sdk_ext.bbclass
> index 9112ab6c5e..14689ec6ac 100644
> --- a/meta/classes/populate_sdk_ext.bbclass
> +++ b/meta/classes/populate_sdk_ext.bbclass
> @@ -251,7 +251,9 @@ python copy_buildsystem () {
> 
>      # Create a layer for new recipes / appends
>      bbpath = d.getVar('BBPATH')
> -    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath',
> baseoutpath, 'create-workspace', '--create-only', 
> os.path.join(baseoutpath,
> 'workspace')])
> +    env = os.environ.copy()
> +    env['PYTHONDONTWRITEBYTECODE'] = '1'
> +    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath',
> baseoutpath, 'create-workspace', '--create-only', 
> os.path.join(baseoutpath,
> 'workspace')], env=env)
> 
>      # Create bblayers.conf
>      bb.utils.mkdirhier(baseoutpath + '/conf')
> diff --git a/meta/lib/oe/copy_buildsystem.py 
> b/meta/lib/oe/copy_buildsystem.py
> index 31a84f5b06..d97bf9d1b9 100644
> --- a/meta/lib/oe/copy_buildsystem.py
> +++ b/meta/lib/oe/copy_buildsystem.py
> @@ -20,7 +20,7 @@ def _smart_copy(src, dest):
>      mode = os.stat(src).st_mode
>      if stat.S_ISDIR(mode):
>          bb.utils.mkdirhier(dest)
> -        cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - 
> -C
> %s -p . \
> +        cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs
> --xattrs-include='*' -chf - -C %s -p . \
>          | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
>          subprocess.check_output(cmd, shell=True, 
> stderr=subprocess.STDOUT)
>      else:
> @@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs,
> input_sstate_cache, output_sstate_cac
>      bb.note('Generating sstate-cache...')
> 
>      nativelsbstring = d.getVar('NATIVELSBSTRING')
> -    bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs,
> input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or 
> ''))
> +    bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s 
> %s
> %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache,
> nativelsbstring, filterfile or ''))
>      if fixedlsbstring and nativelsbstring != fixedlsbstring:
>          nativedir = output_sstate_cache + '/' + nativelsbstring
>          if os.path.isdir(nativedir):
> @@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile,
> cmdprefix='', cwd=None, lo
>          logparam = '-l %s' % logfile
>      else:
>          logparam = ''
> -    cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s 
> -s
> -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
> +    cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1
> PSEUDO_DISABLED=1 oe-check-

[OE-core] [PATCH 0/1] Enable PR Service support in eSDK

[Mark Hatle]

This is the first attempt at enabling the PR service in the eSDK.  It appears
to work for me, but I expect I'm missing corner cases.

Any feedback is appreciatd.

Mark Hatle (1):
  populate_sdk_ext: Add support for PR service

 meta/classes/populate_sdk_ext.bbclass | 25 +
 meta/files/ext-sdk-prepare.py | 17 +
 2 files changed, 42 insertions(+)

-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#149964): 
https://lists.openembedded.org/g/openembedded-core/message/149964
Mute This Topic: https://lists.openembedded.org/mt/81617272/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH 1/1] populate_sdk_ext: Add support for PR service

[Mark Hatle]

From: Mark Hatle 

In the classes/populate_sdk_ext.bbclass the system already copies a number
of configurations, such as the hash equivalency data.  However, the PR
service was being handled.

The new code works by checking if PRSERV_HOST is defined, if it is, use
the existing export functions to write out a conf/prserv.inc file into
the eSDK.  On eSDK install, if a conf/prserv.inc file is present we then
import this file into the system.

This mechanism will work if the PRSERV_HOST is local or remote, as it pulls
the necessary data from the server and then imports it to a local database
on eSDK installation.

Note: the conf/prserv.inc file is not deleted at this time.  It was left
for possible debugging purposes, but removing it is something we could decide
to do in the future.

Signed-off-by: Mark Hatle 
Signed-off-by: Mark Hatle 
---
 meta/classes/populate_sdk_ext.bbclass | 25 +
 meta/files/ext-sdk-prepare.py | 17 +
 2 files changed, 42 insertions(+)

diff --git a/meta/classes/populate_sdk_ext.bbclass 
b/meta/classes/populate_sdk_ext.bbclass
index 14689ec6ac..84232ed9f5 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -375,6 +375,10 @@ python copy_buildsystem () {
 # Map gcc-dependent uninative sstate cache for installer usage
 f.write('SSTATE_MIRRORS += " file://universal/(.*) 
file://universal-4.9/\\1 file://universal-4.9/(.*) 
file://universal-4.8/\\1"\n\n')
 
+if d.getVar("PRSERV_HOST"):
+# Override this, we now include PR data, so it should only 
point ot the local database
+f.write('PRSERV_HOST = "localhost:0"\n\n')
+
 # Allow additional config through sdk-extra.conf
 fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d)
 if fn:
@@ -398,6 +402,27 @@ python copy_buildsystem () {
 bb.utils.mkdirhier(os.path.join(baseoutpath, 'cache'))
 shutil.copyfile(builddir + '/cache/bb_unihashes.dat', baseoutpath + 
'/cache/bb_unihashes.dat')
 
+# If PR Service is in use, we need to export this as well
+bb.note('Do we have a pr database?')
+if d.getVar("PRSERV_HOST"):
+bb.note('Writing PR database...')
+# Based on the code in classes/prexport.bbclass
+import oe.prservice
+#dump meta info of tables
+localdata = d.createCopy()
+localdata.setVar('PRSERV_DUMPOPT_COL', "1")
+localdata.setVar('PRSERV_DUMPDIR', os.path.join(baseoutpath, 'conf'))
+localdata.setVar('PRSERV_DUMPFILE', '${PRSERV_DUMPDIR}/prserv.inc')
+
+bb.note('PR Database write to %s' % 
(localdata.getVar('PRSERV_DUMPFILE')))
+
+retval = oe.prservice.prserv_dump_db(localdata)
+if not retval:
+bb.error("prexport_handler: export failed!")
+return
+(metainfo, datainfo) = retval
+oe.prservice.prserv_export_tofile(localdata, metainfo, datainfo, True)
+
 # Use templateconf.cfg file from builddir if exists
 if os.path.exists(builddir + '/conf/templateconf.cfg') and 
use_custom_templateconf == '1':
 shutil.copyfile(builddir + '/conf/templateconf.cfg', baseoutpath + 
'/conf/templateconf.cfg')
diff --git a/meta/files/ext-sdk-prepare.py b/meta/files/ext-sdk-prepare.py
index 163d5e9912..d191e5e19c 100644
--- a/meta/files/ext-sdk-prepare.py
+++ b/meta/files/ext-sdk-prepare.py
@@ -44,6 +44,23 @@ def main():
 sdk_targets = []
 else:
 sdk_targets = ' '.join(sys.argv[2:]).split()
+
+prserv = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conf', 
'prserv.inc')
+if os.path.isfile(prserv):
+with open(logfile, 'a') as logf:
+logf.write('Importing PR data...\n')
+
+ret = run_command_interruptible('bitbake-prserv-tool import %s' % 
prserv)
+
+lastlog = get_last_consolelog()
+if lastlog:
+with open(lastlog, 'r') as f:
+for line in f:
+logf.write(line)
+if ret:
+print('ERROR: PR data import failed: error log written to %s' 
% logfile)
+return ret
+
 if not sdk_targets:
 # Just do a parse so the cache is primed
 ret = run_command_interruptible('bitbake -p --quiet')
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#149963): 
https://lists.openembedded.org/g/openembedded-core/message/149963
Mute This Topic: https://lists.openembedded.org/mt/81617271/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] master eSDK hash problem with METADATA_REVISION and base-files:do_install

[Mark Hatle]

Building an eSDK (all stock config):

MACHINE=qemuarm bitbake core-image-minimal -c populate_sdk_ext


I then install it, and get:

Checking sstate mirror object availability: 100%
|#|
Time: 0:00:00
WARNING: The base-files:do_install sig is computed to be
587ccfa9299e26094ae342f1ce3d782bfaf776c639013428a511770c3828d49a, but the sig is
locked to 0c35f266f1e79b1f34260df1b8d677f7eb74a7f7241155601ed73a014c8e286c in
SIGGEN_LOCKEDSIGS_t-qemuarm
Loading cache: 100%
|##|
Time: 0:00:00
Initialising tasks: 100%
|#|
Time: 0:00:00


Running a bitbake diffsigs on this file, I get:

basehash changed from
cffec52acb67f62fe3812d0d42d52b1081f9150386f43f0426f5469b4a1a6a25 to
332259212c5e9a4ed2a7fe25b8ea77cb54fb6b82f7deed90179e145f1d0cf1b6
Variable METADATA_REVISION value changed from
'3210cabe56d6cf6d51638e6f31f93110e40ab2cd' to ''


So it appears that some of the recent work on METADATA_REVISION has broken hash
computations in the base-files package due to the value being 'unknown' in one,
and set in the other.

--Mark

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#149961): 
https://lists.openembedded.org/g/openembedded-core/message/149961
Mute This Topic: https://lists.openembedded.org/mt/81616842/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH] populate_sdk_ext: Avoid copying and producing .pyc files

[Mark Hatle]

Since pyc cache files are really system specific, no real reason to copy or
generate them during the eSDK build process.  Also generating them has the
possibility of re-using inodes that pseudo may have been tracking, leading
a build failure.

Signed-off-by: Mark Hatle 
---
 meta/classes/populate_sdk_ext.bbclass | 4 +++-
 meta/lib/oe/copy_buildsystem.py   | 6 +++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/meta/classes/populate_sdk_ext.bbclass 
b/meta/classes/populate_sdk_ext.bbclass
index 9112ab6c5e..14689ec6ac 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -251,7 +251,9 @@ python copy_buildsystem () {
 
 # Create a layer for new recipes / appends
 bbpath = d.getVar('BBPATH')
-bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 
'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')])
+env = os.environ.copy()
+env['PYTHONDONTWRITEBYTECODE'] = '1'
+bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 
'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')], 
env=env)
 
 # Create bblayers.conf
 bb.utils.mkdirhier(baseoutpath + '/conf')
diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buildsystem.py
index 31a84f5b06..d97bf9d1b9 100644
--- a/meta/lib/oe/copy_buildsystem.py
+++ b/meta/lib/oe/copy_buildsystem.py
@@ -20,7 +20,7 @@ def _smart_copy(src, dest):
 mode = os.stat(src).st_mode
 if stat.S_ISDIR(mode):
 bb.utils.mkdirhier(dest)
-cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s 
-p . \
+cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs 
--xattrs-include='*' -chf - -C %s -p . \
 | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
 else:
@@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs, 
input_sstate_cache, output_sstate_cac
 bb.note('Generating sstate-cache...')
 
 nativelsbstring = d.getVar('NATIVELSBSTRING')
-bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, 
input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
+bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s %s 
%s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, 
filterfile or ''))
 if fixedlsbstring and nativelsbstring != fixedlsbstring:
 nativedir = output_sstate_cache + '/' + nativelsbstring
 if os.path.isdir(nativedir):
@@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile, 
cmdprefix='', cwd=None, lo
 logparam = '-l %s' % logfile
 else:
 logparam = ''
-cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o 
%s %s" % (cmdprefix, targets, filteroutfile, logparam)
+cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 
oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
 env = dict(d.getVar('BB_ORIGENV', False))
 env.pop('BUILDDIR', '')
 env.pop('BBPATH', '')
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#149960): 
https://lists.openembedded.org/g/openembedded-core/message/149960
Mute This Topic: https://lists.openembedded.org/mt/81615695/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] any *compelling* reasons to whitelist env vars for an OE build?

[Mark Hatle]

On 3/25/21 9:08 AM, Robert P. J. Day wrote:
> 
>   kind of a philosophical question, but i'm having a discussion with
> some new colleagues about how to customize an OE (actually, wind river
> linux) build, and these colleagues have, until now, used (whitelisted)
> environment variables to pass info to the build, that info being used
> to specify things like a development versus manufacturing build, and
> so on.
> 
>   i suggested that i didn't think there was any really persuasive
> reasons to use environment variables this way, as there are more than
> enough configuration files to customize a build. i mentioned things
> like:
> 
>   * machine config files
>   * distro config files
>   * defining packagegroup files
>   * defining image files
> 
> and on and on and on.
> 
>   my point was that there are more than enough ways to support all the
> customization you might need, that taking advantage of shell
> environment variables strikes me as unnecessarily messy and, for that
> matter, dangerous if you forget so set the environment.
> 
>   thoughts? am i out of line in advising them to use what OE offers
> them, and not extract stuff from the environment?

I _always_ do builds like:

MACHINE=zynqmp-generic DISTRO=petalinux bitbake core-image-minimal

I don't want to have to update any configuration files to pass basic values into
the system.  This is also used heavily on my build systems so we can have a
common configuration for all builds, but built across a wide breadth of 
components.

I also occasionally use BB_ENV_EXTRAWHITE to add additional variables I might
want to override.  For instance, sometimes I want TMPDIR to go to a different
location, instead of having to modify local.conf (and then remember to put it
back the way it was) I can just add to the EXTRAWHITE, and call passing in 
TMPDIR.

--Mark

> rday
> 
> 
> 
> 
> 

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#149928): 
https://lists.openembedded.org/g/openembedded-core/message/149928
Mute This Topic: https://lists.openembedded.org/mt/81603196/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] PR Service and eSDK

[Mark Hatle]

On 3/23/21 5:45 PM, Richard Purdie wrote:
> On Tue, 2021-03-23 at 17:34 -0500, Mark Hatle wrote:
>> For some reason I thought if PR service was enabled, when the eSDK was 
>> generated
>> it would export the pr service information and include it within the eSDK.
>>
>> I'm not finding the file, or even code that does this.  Am I having a fever
>> dream, or is there code that should be doing this?
>>
>> What I'm trying to build is an eSDK that the user can use devtool and 
>> construct
>> a new version of a package, then they can deploy the new package within their
>> package feed to update their targets.  Without the PR service in the eSDK, 
>> there
>> won't be any way to get a correct PR number.
> 
> I know we copy bb_unihashes.dat into the eSDK with code in 
> populate_sdl_ext.bbclass
> for this reason for hashequiv.
> 
> We do have prserv export and import code but I'm not sure anyone has 
> integrated it
> with eSDK, or if there is integration, I can't spot it atm. I wondered if it 
> copied
> the prserv database from the build directory over to assist with this issue 
> but
> again, I can't spot it.

Ok, this confirms what I am seeing then.  Maybe it was discussed and was never
implemented.  I'll look at where the unihashes.dat is copied over and look to do
the export... and then import in the setup script.

--Mark

> Cheers,
> 
> Richard
> 
> 
> 
> 
> 

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#149851): 
https://lists.openembedded.org/g/openembedded-core/message/149851
Mute This Topic: https://lists.openembedded.org/mt/81563805/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] PR Service and eSDK

[Mark Hatle]

For some reason I thought if PR service was enabled, when the eSDK was generated
it would export the pr service information and include it within the eSDK.

I'm not finding the file, or even code that does this.  Am I having a fever
dream, or is there code that should be doing this?

What I'm trying to build is an eSDK that the user can use devtool and construct
a new version of a package, then they can deploy the new package within their
package feed to update their targets.  Without the PR service in the eSDK, there
won't be any way to get a correct PR number.

--Mark

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#149849): 
https://lists.openembedded.org/g/openembedded-core/message/149849
Mute This Topic: https://lists.openembedded.org/mt/81563805/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH 1/1] extrausers: Add ability to force password change on first login

[Mark Hatle]

On 3/8/21 8:02 PM, Chen Qi wrote:
> Hi Mark,
> 
> Is it something similar to 'passwd-expire' in this extrausers.bbclass?

I wasn't aware of that evening existing.  Yes it looks like it does the same 
thing.

I can withdraw my change then, but we may want to considering adding something
to the documentation about security practices.  For accounts that are created by
the build system, it's best practices to either not make them able to be logged
in with (login locked out '-P *' on the adduser) or force the password to be
reset on next login (using passwd-expire).

--Mark

> Best Regards,
> Chen Qi
> 
> On 03/09/2021 02:08 AM, Mark Hatle wrote:
>> As documented in shadow(5), the third parameter is the last login time.  A
>> special value of '0' is defined which causes the password system to force
>> a password change on next login.
>>
>> Adding the variable "EXTRA_FORCE_PASSWORD_CHANGE", a space separated list of
>> user names, we can use this to adjust the shadow file's third value for the
>> listed users.
>>
>> Note: This does have the same dependencies as other usages of extrausers,
>> specifically base-passwd and shadow.
>>
>> Signed-off-by: Mark Hatle 
>> Signed-off-by: Mark Hatle 
>> ---
>>  meta/classes/extrausers.bbclass | 29 +++--
>>  meta/conf/documentation.conf|  1 +
>>  2 files changed, 28 insertions(+), 2 deletions(-)
>>
>> diff --git a/meta/classes/extrausers.bbclass 
>> b/meta/classes/extrausers.bbclass
>> index 90811bfe2a..e9d9358bef 100644
>> --- a/meta/classes/extrausers.bbclass
>> +++ b/meta/classes/extrausers.bbclass
>> @@ -14,10 +14,10 @@
>>  
>>  inherit useradd_base
>>  
>> -PACKAGE_INSTALL_append = " ${@['', 'base-passwd 
>> shadow'][bool(d.getVar('EXTRA_USERS_PARAMS'))]}"
>> +PACKAGE_INSTALL_append = " ${@['', 'base-passwd 
>> shadow'][bool(d.getVar('EXTRA_USERS_PARAMS')) or 
>> bool(d.getVar('EXTRA_FORCE_PASSWORD_CHANGE'))]}"
>>  
>>  # Image level user / group settings
>> -ROOTFS_POSTPROCESS_COMMAND_append = " set_user_group;"
>> +ROOTFS_POSTPROCESS_COMMAND_append = "${@['', ' 
>> set_user_group;'][bool(d.getVar('EXTRA_USERS_PARAMS'))]}"
>>  
>>  # Image level user / group settings
>>  set_user_group () {
>> @@ -66,6 +66,31 @@ set_user_group () {
>>  done
>>  }
>>  
>> +# Image level force a specific user/users to reset their password on first 
>> login
>> +# Note: this requires shadow passwords and login programs that respect the 
>> shadow
>> +# expiration field.
>> +ROOTFS_POSTPROCESS_COMMAND_append = "${@['', '
>> force_password_change;'][bool(d.getVar('EXTRA_FORCE_PASSWORD_CHANGE'))]}"
>> +
>> +# Works by setting 'date of last password change' to 0, which has a special
>> +# meaning of 'user should change her password the next time she will log in 
>> the
>> +# system' See: shadow (5)
>> +force_password_change () {
>> +if [ ! -e ${IMAGE_ROOTFS}/etc/shadow ]; then
>> +bberror "/etc/shadow does not exist in the image, unable to set 
>> password change on login."
>> +return
>> +fi
>> +passwd_change_users="${EXTRA_FORCE_PASSWORD_CHANGE}"
>> +export PSEUDO="${FAKEROOTENV} ${STAGING_DIR_NATIVE}${bindir}/pseudo"
>> +for name in $passwd_change_users; do
>> +if ! grep -q '^'$name':' ${IMAGE_ROOTFS}/etc/shadow ; then
>> +bberror "Unable to find user $name in /etc/shadow, 
>> unable to set password change on login."
>> +fi
>> +bbnote "Set user $name to need a password change on first 
>> login."
>> +cmd="sed -i ${IMAGE_ROOTFS}/etc/shadow -e 
>> 's,^'$name':\\([^:]*\\):[^:]*:,'$name':\\1:0:,'"
>> +eval flock -x ${IMAGE_ROOTFS}${sysconfdir} -c \"$PSEUDO $cmd\" 
>> || true
>> +done
>> +}
>> +
>>  USERADDEXTENSION ?= ""
>>  
>>  inherit ${USERADDEXTENSION}
>> diff --git a/meta/conf/documentation.conf b/meta/conf/documentation.conf
>> index c5a38b0764..d1c5b8b1a3 100644
>> --- a/meta/conf/documentation.conf
>> +++ b/meta/conf/documentation.conf
>> @@ -169,6 +169,7 @@ EXTRA_OESCONS[doc] = "When a recipe inherits the scons 
>> class, this variable spec
>>  EXTRA_QMAKEVARS_POST[doc] = "Configuration variables or options you want to 
>> pass to qmake when the arguments need to be after the .pro file list on the 
>> command line."
&

Re: [OE-core] [PATCH 1/1] extrausers: Add ability to force password change on first login

[Mark Hatle]

On 3/8/21 12:50 PM, Khem Raj wrote:
> 
> 
> On 3/8/21 10:08 AM, Mark Hatle wrote:
>> From: Mark Hatle 
>>
>> As documented in shadow(5), the third parameter is the last login time.  A
>> special value of '0' is defined which causes the password system to force
>> a password change on next login.
>>
>> Adding the variable "EXTRA_FORCE_PASSWORD_CHANGE", a space separated list of
>> user names, we can use this to adjust the shadow file's third value for the
>> listed users.
>>
>> Note: This does have the same dependencies as other usages of extrausers,
>> specifically base-passwd and shadow.
>>
> 
> I think it should check for r/w rootfs feature perhaps. unrelated to 

Is there a standard way to check for a r/w roots?  If there is, easy to add.

> this change but it seems it adds a dep on shadow disregarding DISTRO 
> policies where user might have chosen a different login managager, it 
> should perhaps warn about it.

The dep on shadow is the same as any extrauser call.  The dependency sets the
minimum login manager, but any login manager that supports proper shadow
password handling will work.  If it doesn't support shadow password handling
then nothing breaks -- it just won't do anything.  (Really nothing here that can
be enforced in this code block.)

util-linux login + pam for instance used to work.  (I've not tested it though in
a few years.)

--Mark

>> Signed-off-by: Mark Hatle 
>> Signed-off-by: Mark Hatle 
>> ---
>>   meta/classes/extrausers.bbclass | 29 +++--
>>   meta/conf/documentation.conf|  1 +
>>   2 files changed, 28 insertions(+), 2 deletions(-)
>>
>> diff --git a/meta/classes/extrausers.bbclass 
>> b/meta/classes/extrausers.bbclass
>> index 90811bfe2a..e9d9358bef 100644
>> --- a/meta/classes/extrausers.bbclass
>> +++ b/meta/classes/extrausers.bbclass
>> @@ -14,10 +14,10 @@
>>   
>>   inherit useradd_base
>>   
>> -PACKAGE_INSTALL_append = " ${@['', 'base-passwd 
>> shadow'][bool(d.getVar('EXTRA_USERS_PARAMS'))]}"
>> +PACKAGE_INSTALL_append = " ${@['', 'base-passwd 
>> shadow'][bool(d.getVar('EXTRA_USERS_PARAMS')) or 
>> bool(d.getVar('EXTRA_FORCE_PASSWORD_CHANGE'))]}"
>>   
>>   # Image level user / group settings
>> -ROOTFS_POSTPROCESS_COMMAND_append = " set_user_group;"
>> +ROOTFS_POSTPROCESS_COMMAND_append = "${@['', ' 
>> set_user_group;'][bool(d.getVar('EXTRA_USERS_PARAMS'))]}"
>>   
>>   # Image level user / group settings
>>   set_user_group () {
>> @@ -66,6 +66,31 @@ set_user_group () {
>>  done
>>   }
>>   
>> +# Image level force a specific user/users to reset their password on first 
>> login
>> +# Note: this requires shadow passwords and login programs that respect the 
>> shadow
>> +# expiration field.
>> +ROOTFS_POSTPROCESS_COMMAND_append = "${@['', ' 
>> force_password_change;'][bool(d.getVar('EXTRA_FORCE_PASSWORD_CHANGE'))]}"
>> +
>> +# Works by setting 'date of last password change' to 0, which has a special
>> +# meaning of 'user should change her password the next time she will log in 
>> the
>> +# system' See: shadow (5)
>> +force_password_change () {
>> +if [ ! -e ${IMAGE_ROOTFS}/etc/shadow ]; then
>> +bberror "/etc/shadow does not exist in the image, unable to set 
>> password change on login."
>> +return
>> +fi
>> +passwd_change_users="${EXTRA_FORCE_PASSWORD_CHANGE}"
>> +export PSEUDO="${FAKEROOTENV} ${STAGING_DIR_NATIVE}${bindir}/pseudo"
>> +for name in $passwd_change_users; do
>> +if ! grep -q '^'$name':' ${IMAGE_ROOTFS}/etc/shadow ; then
>> +bberror "Unable to find user $name in /etc/shadow, 
>> unable to set password change on login."
>> +fi
>> +bbnote "Set user $name to need a password change on first 
>> login."
>> +cmd="sed -i ${IMAGE_ROOTFS}/etc/shadow -e 
>> 's,^'$name':\\([^:]*\\):[^:]*:,'$name':\\1:0:,'"
>> +eval flock -x ${IMAGE_ROOTFS}${sysconfdir} -c \"$PSEUDO $cmd\" 
>> || true
>> +done
>> +}
>> +
>>   USERADDEXTENSION ?= ""
>>   
>>   inherit ${USERADDEXTENSION}
>> diff --git a/meta/conf/documentation.conf b/meta/conf/documentation.conf
>> index c5a38b0764..d1c5b8b1a3 100644
>> --- a/meta/conf/documentation.conf
>> +++ b/meta/conf/documentation.conf
>> @@ -169,6 +169,7 @@ EXTRA_OESCONS[doc] = "When a rec