commit: 9ac3f1cf62b522236ba9efd7e923071c37df1e6d
Author: Sam James <sam <AT> gentoo <DOT> org>
AuthorDate: Sat Jul 30 04:46:15 2022 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sat Jul 30 05:21:25 2022 +0000
URL: https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=9ac3f1cf
toolchain.eclass: store previous test results & compare in src_test
This is what GCC developers do, so let's give it a try. Note that
folks will have to (for now?) set GCC_TESTS_IGNORE_NO_BASELINE=1
for it to succeed if src_test failed for the first time & they don't
have any test data saved for that slot.
The test data is stored in ${EPREFIX}/var/cache/gcc/${SLOT}. GCC
upstream's contrib/compare_tests script performs the comparison.
We may want to investigate shipping baseline results / gathering
them in a repo somewhere.
We may also in future want to install this data as orphaned
so that it's kept around and not lost when upgrading versions
and to allow cross-slot comparison when an old one is depcleaned
(although I guess doesn't matter so much if it's first time, but still).
Signed-off-by: Sam James <sam <AT> gentoo.org>
eclass/toolchain.eclass | 44 ++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 42 insertions(+), 2 deletions(-)
diff --git a/eclass/toolchain.eclass b/eclass/toolchain.eclass
index caabb5f25fa1..d4007968ac8a 100644
--- a/eclass/toolchain.eclass
+++ b/eclass/toolchain.eclass
@@ -1882,12 +1882,39 @@ toolchain_src_test() {
# 'asan' wants to be preloaded first, so does 'sandbox'.
# To make asan tests work disable sandbox for all of test suite.
# 'backtrace' tests also does not like 'libsandbox.so' presence.
- SANDBOX_ON=0 LD_PRELOAD= emake -k check
+ #
+ # Nonfatal here as we die if compare_tests failed
+ SANDBOX_ON=0 LD_PRELOAD= nonfatal emake -k check
+ local success_tests=$?
+
+ if [[ ! -d "${BROOT}"/var/cache/gcc/${SLOT} ]] && ! [[ ${success_tests}
-eq 0 ]] ; then
+ # We have no reference data saved from a previous run to know if
+ # the failures are tolerable or not, so we bail out.
+ eerror "Reference test data does NOT exist at
${BROOT}/var/cache/gcc/${SLOT}"
+ eerror "Tests failed and nothing to compare with, so this is a
fatal error."
+ eerror "(Set GCC_TESTS_IGNORE_NO_BASELINE=1 to make this
non-fatal for initial run.)"
+
+ if [[ -z ${GCC_TESTS_IGNORE_NO_BASELINE} ]] ; then
+ die "Tests failed (failures occurred with no reference
data)"
+ fi
+ fi
- einfo "Testing complete."
+ einfo "Testing complete! Review the following output to check for
success or failure."
einfo "Please ignore any 'mail' lines in the summary output below (no
mail is sent)."
einfo "Summary:"
"${S}"/contrib/test_summary
+
+ # If previous results exist on the system, compare with it
+ # TODO: Distribute some baseline results in e.g. gcc-patches.git?
+ if [[ -d "${BROOT}"/var/cache/gcc/${SLOT} ]] ; then
+ einfo "Comparing with previous cached results at
${BROOT}/var/cache/gcc/${SLOT}"
+
+ # Exit with the following values:
+ # 0 if there is nothing of interest
+ # 1 if there are errors when comparing single test case files
+ # N for the number of errors found when comparing directories
+ "${S}"/contrib/compare_tests "${BROOT}"/var/cache/gcc/${SLOT}/
. || die "Comparison for tests results failed, error code: $?"
+ fi
}
#---->> src_install <<----
@@ -2123,6 +2150,19 @@ toolchain_src_install() {
pax-mark -m
"${ED}/libexec/gcc/${CTARGET}/${GCC_CONFIG_VER}/ecj1"
pax-mark -m "${ED}/${CTARGET}/gcc-bin/${GCC_CONFIG_VER}/gij"
fi
+
+ if use test ; then
+ # TODO: In future, install orphaned to allow comparison across
+ # more versions even after unmerged? Also would be useful for
+ # historical records and tracking down regressions a while
+ # after they first appeared, but were only just reported.
+ einfo "Copying test results to ${EPREFIX}/var/cache/gcc/${SLOT}
for future comparison"
+ (
+ dodir /var/cache/gcc/${SLOT}
+ cd "${WORKDIR}"/build || die
+ find . -name \*.sum -exec cp --parents -v {}
"${ED}"/var/cache/gcc/${SLOT} \;
+ )
+ fi
}
# Move around the libs to the right location. For some reason,
