Re: [OE-core] [PATCH v3 4/6] oeqa/selftest/resulttool: add test for metadata filtering on regression

2023-02-25 Thread Richard Purdie 
On Fri, 2023-02-24 at 17:45 +0100, Alexis Lothoré via
lists.openembedded.org wrote:
> From: Alexis Lothoré 
> 
> Introduce new tests for the metadata-based filtering added for oeselftest
> results
> 
> Signed-off-by: Alexis Lothoré 
> ---
>  .../oeqa/selftest/cases/resulttooltests.py| 137 ++
>  1 file changed, 137 insertions(+)
> 
> diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py 
> b/meta/lib/oeqa/selftest/cases/resulttooltests.py
> index efdfd98af3c..75d406c122d 100644
> --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py
> +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py
> @@ -98,3 +98,140 @@ class ResultToolTests(OESelftestTestCase):
>  resultutils.append_resultsdata(results, 
> ResultToolTests.target_results_data, configmap=resultutils.flatten_map)
>  self.assertEqual(len(results[''].keys()), 5, msg="Flattened results 
> not correct %s" % str(results))
>  
> +def test_results_without_metadata_can_be_compared(self):
> +base_configuration = {"TEST_TYPE": "oeselftest",
> +  "TESTSERIES": "series1",
> +  "IMAGE_BASENAME": "image",
> +  "IMAGE_PKGTYPE": "ipk",
> +  "DISTRO": "mydistro",
> +  "MACHINE": "qemux86"}
> +target_configuration = {"TEST_TYPE": "oeselftest",
> +"TESTSERIES": "series1",
> +"IMAGE_BASENAME": "image",
> +"IMAGE_PKGTYPE": "ipk",
> +"DISTRO": "mydistro",
> +"MACHINE": "qemux86"}
> +self.assertTrue(regression.can_be_compared(base_configuration, 
> target_configuration),
> +msg="incorrect metadata filtering, tests without 
> metadata should be compared")
> +
> +def test_target_result_with_missing_metadata_can_not_be_compared(self):
> +base_configuration = {"TEST_TYPE": "oeselftest",
> +  "TESTSERIES": "series1",
> +  "IMAGE_BASENAME": "image",
> +  "IMAGE_PKGTYPE": "ipk",
> +  "DISTRO": "mydistro",
> +  "MACHINE": "qemux86",
> +  "OESELFTEST_METADATA": {"run_all_tests": True,
> +  "run_tests": None,
> +  "skips": None,
> +  "machine": None,
> +  "select_tags": 
> ["toolchain-user", "toolchain-system"],
> +  "exclude_tags": None}}
> +target_configuration = {"TEST_TYPE": "oeselftest",
> +"TESTSERIES": "series1",
> +"IMAGE_BASENAME": "image",
> +"IMAGE_PKGTYPE": "ipk",
> +"DISTRO": "mydistro",
> +"MACHINE": "qemux86"}
> +self.assertFalse(regression.can_be_compared(base_configuration, 
> target_configuration),
> + msg="incorrect metadata filtering, tests should not 
> be compared")
> +
> +def test_results_with_matching_metadata_can_be_compared(self):
> +base_configuration = {"TEST_TYPE": "oeselftest",
> +  "TESTSERIES": "series1",
> +  "IMAGE_BASENAME": "image",
> +  "IMAGE_PKGTYPE": "ipk",
> +  "DISTRO": "mydistro",
> +  "MACHINE": "qemux86",
> +  "OESELFTEST_METADATA": {"run_all_tests": True,
> +  "run_tests": None,
> +  "skips": None,
> +  "machine": None,
> +  "select_tags": 
> ["toolchain-user", "toolchain-system"],
> +  "exclude_tags": None}}
> +target_configuration = {"TEST_TYPE": "oeselftest",
> +"TESTSERIES": "series1",
> +"IMAGE_BASENAME": "image",
> +"IMAGE_PKGTYPE": "ipk",
> +"DISTRO": "mydistro",
> +"MACHINE": "qemux86",
> +"OESELFTEST_METADATA": {"run_all_tests": 
> True,
> +"run_tests": None,
> +"skips": None,
> +"machine": None,
> +

[OE-core] [PATCH v3 4/6] oeqa/selftest/resulttool: add test for metadata filtering on regression

2023-02-24 Thread Alexis Lothoré via lists . openembedded . org 
From: Alexis Lothoré 

Introduce new tests for the metadata-based filtering added for oeselftest
results

Signed-off-by: Alexis Lothoré 
---
 .../oeqa/selftest/cases/resulttooltests.py| 137 ++
 1 file changed, 137 insertions(+)

diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py 
b/meta/lib/oeqa/selftest/cases/resulttooltests.py
index efdfd98af3c..75d406c122d 100644
--- a/meta/lib/oeqa/selftest/cases/resulttooltests.py
+++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py
@@ -98,3 +98,140 @@ class ResultToolTests(OESelftestTestCase):
 resultutils.append_resultsdata(results, 
ResultToolTests.target_results_data, configmap=resultutils.flatten_map)
 self.assertEqual(len(results[''].keys()), 5, msg="Flattened results 
not correct %s" % str(results))
 
+def test_results_without_metadata_can_be_compared(self):
+base_configuration = {"TEST_TYPE": "oeselftest",
+  "TESTSERIES": "series1",
+  "IMAGE_BASENAME": "image",
+  "IMAGE_PKGTYPE": "ipk",
+  "DISTRO": "mydistro",
+  "MACHINE": "qemux86"}
+target_configuration = {"TEST_TYPE": "oeselftest",
+"TESTSERIES": "series1",
+"IMAGE_BASENAME": "image",
+"IMAGE_PKGTYPE": "ipk",
+"DISTRO": "mydistro",
+"MACHINE": "qemux86"}
+self.assertTrue(regression.can_be_compared(base_configuration, 
target_configuration),
+msg="incorrect metadata filtering, tests without 
metadata should be compared")
+
+def test_target_result_with_missing_metadata_can_not_be_compared(self):
+base_configuration = {"TEST_TYPE": "oeselftest",
+  "TESTSERIES": "series1",
+  "IMAGE_BASENAME": "image",
+  "IMAGE_PKGTYPE": "ipk",
+  "DISTRO": "mydistro",
+  "MACHINE": "qemux86",
+  "OESELFTEST_METADATA": {"run_all_tests": True,
+  "run_tests": None,
+  "skips": None,
+  "machine": None,
+  "select_tags": 
["toolchain-user", "toolchain-system"],
+  "exclude_tags": None}}
+target_configuration = {"TEST_TYPE": "oeselftest",
+"TESTSERIES": "series1",
+"IMAGE_BASENAME": "image",
+"IMAGE_PKGTYPE": "ipk",
+"DISTRO": "mydistro",
+"MACHINE": "qemux86"}
+self.assertFalse(regression.can_be_compared(base_configuration, 
target_configuration),
+ msg="incorrect metadata filtering, tests should not 
be compared")
+
+def test_results_with_matching_metadata_can_be_compared(self):
+base_configuration = {"TEST_TYPE": "oeselftest",
+  "TESTSERIES": "series1",
+  "IMAGE_BASENAME": "image",
+  "IMAGE_PKGTYPE": "ipk",
+  "DISTRO": "mydistro",
+  "MACHINE": "qemux86",
+  "OESELFTEST_METADATA": {"run_all_tests": True,
+  "run_tests": None,
+  "skips": None,
+  "machine": None,
+  "select_tags": 
["toolchain-user", "toolchain-system"],
+  "exclude_tags": None}}
+target_configuration = {"TEST_TYPE": "oeselftest",
+"TESTSERIES": "series1",
+"IMAGE_BASENAME": "image",
+"IMAGE_PKGTYPE": "ipk",
+"DISTRO": "mydistro",
+"MACHINE": "qemux86",
+"OESELFTEST_METADATA": {"run_all_tests": True,
+"run_tests": None,
+"skips": None,
+"machine": None,
+"select_tags": 
["toolchain-user", "toolchain-system"],
+"exclude_tags": None}}
+self.assertTrue(regression.can_be_compared(base_configuration, 
target_configuration),
+