commit: 7bbfb01a32b73842f8908de028703510a0e12057 Author: Sam James <sam <AT> gentoo <DOT> org> AuthorDate: Sat Mar 16 08:06:55 2024 +0000 Commit: Sam James <sam <AT> gentoo <DOT> org> CommitDate: Sat Mar 16 08:06:55 2024 +0000 URL: https://gitweb.gentoo.org/proj/gcc-patches.git/commit/?id=7bbfb01a
scripts: import testsuite-management/validate_failures.py from GCC Imported from GCC commit 53fb2cf75965e4dbcf145a12d8ae41f4667a8498. We want the latest version to use in ebuilds. Signed-off-by: Sam James <sam <AT> gentoo.org> scripts/README | 1 + scripts/testsuite-management/validate_failures.py | 733 +++++++++++++++++++++ .../x86_64-unknown-linux-gnu.xfail | 109 +++ 3 files changed, 843 insertions(+) diff --git a/scripts/README b/scripts/README new file mode 100644 index 0000000..06c5f91 --- /dev/null +++ b/scripts/README @@ -0,0 +1 @@ +testsuite-management is imported from gcc.git (contrib/testsuite-management) diff --git a/scripts/testsuite-management/validate_failures.py b/scripts/testsuite-management/validate_failures.py new file mode 100755 index 0000000..f81ac4f --- /dev/null +++ b/scripts/testsuite-management/validate_failures.py @@ -0,0 +1,733 @@ +#!/usr/bin/env python3 + +# Script to compare testsuite failures against a list of known-to-fail +# tests. + +# Contributed by Diego Novillo <[email protected]> +# +# Copyright (C) 2011-2024 Free Software Foundation, Inc. +# +# This file is part of GCC. +# +# GCC is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# GCC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GCC; see the file COPYING. If not, write to +# the Free Software Foundation, 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + +"""This script provides a coarser XFAILing mechanism that requires no +detailed DejaGNU markings. This is useful in a variety of scenarios: + +- Development branches with many known failures waiting to be fixed. +- Release branches with known failures that are not considered + important for the particular release criteria used in that branch. + +The script must be executed from the toplevel build directory. When +executed it will: + +1- Determine the target built: TARGET +2- Determine the source directory: SRCDIR +3- Look for a failure manifest file in + <SRCDIR>/<MANIFEST_SUBDIR>/<MANIFEST_NAME>.xfail +4- Collect all the <tool>.sum files from the build tree. +5- Produce a report stating: + a- Failures expected in the manifest but not present in the build. + b- Failures in the build not expected in the manifest. +6- If all the build failures are expected in the manifest, it exits + with exit code 0. Otherwise, it exits with error code 1. + +Manifest files contain expected DejaGNU results that are otherwise +treated as failures. +They may also contain additional text: + +# This is a comment. - self explanatory +@include file - the file is a path relative to the includer +@remove result text - result text is removed from the expected set +""" + +import datetime +import optparse +import os +import re +import sys + +_VALID_TEST_RESULTS = [ 'FAIL', 'UNRESOLVED', 'XPASS', 'ERROR' ] +# <STATE>: <NAME> <DESCRIPTION" +_VALID_TEST_RESULTS_REX = re.compile('(%s):\s*(\S+)\s*(.*)' + % "|".join(_VALID_TEST_RESULTS)) + +# Formats of .sum file sections +_TOOL_LINE_FORMAT = '\t\t=== %s tests ===\n' +_EXP_LINE_FORMAT = '\nRunning %s:%s ...\n' +_SUMMARY_LINE_FORMAT = '\n\t\t=== %s Summary ===\n' + +# ... and their compiled regexs. +_TOOL_LINE_REX = re.compile('^\t\t=== (.*) tests ===\n') +# Match .exp file name, optionally prefixed by a "tool:" name and a +# path ending with "testsuite/" +_EXP_LINE_REX = re.compile('^Running (?:.*:)?(.*) \.\.\.\n') +_SUMMARY_LINE_REX = re.compile('^\t\t=== (.*) Summary ===\n') + +# Subdirectory of srcdir in which to find the manifest file. +_MANIFEST_SUBDIR = 'contrib/testsuite-management' + +# Pattern for naming manifest files. +# The first argument should be the toplevel GCC(/GNU tool) source directory. +# The second argument is the manifest subdir. +# The third argument is the manifest target, which defaults to the target +# triplet used during the build. +_MANIFEST_PATH_PATTERN = '%s/%s/%s.xfail' + +# The options passed to the program. +_OPTIONS = None + +def Error(msg): + print('error: %s' % msg, file=sys.stderr) + sys.exit(1) + + +class TestResult(object): + """Describes a single DejaGNU test result as emitted in .sum files. + + We are only interested in representing unsuccessful tests. So, only + a subset of all the tests are loaded. + + The summary line used to build the test result should have this format: + + attrlist | XPASS: gcc.dg/unroll_1.c (test for excess errors) + ^^^^^^^^ ^^^^^ ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^ + optional state name description + attributes + + Attributes: + attrlist: A comma separated list of attributes. + Valid values: + flaky Indicates that this test may not always fail. These + tests are reported, but their presence does not affect + the results. + + expire=YYYYMMDD After this date, this test will produce an error + whether it is in the manifest or not. + + state: One of UNRESOLVED, XPASS or FAIL. + name: File name for the test. + description: String describing the test (flags used, dejagnu message, etc) + ordinal: Monotonically increasing integer. + It is used to keep results for one .exp file sorted + by the order the tests were run. + tool: Top-level testsuite name (aka "tool" in DejaGnu parlance) of the test. + exp: Name of .exp testsuite file. + """ + + def __init__(self, summary_line, ordinal, tool, exp): + try: + (self.attrs, summary_line) = SplitAttributesFromSummaryLine(summary_line) + try: + (self.state, + self.name, + self.description) = _VALID_TEST_RESULTS_REX.match(summary_line).groups() + if _OPTIONS.srcpath_regex and _OPTIONS.srcpath_regex != '': + self.description = re.sub(_OPTIONS.srcpath_regex, '', + self.description) + except: + print('Failed to parse summary line: "%s"' % summary_line, + file=sys.stderr) + raise + self.ordinal = ordinal + if tool == None or exp == None: + # .sum file seem to be broken. There was no "tool" and/or "exp" + # lines preceding this result. + print(f'.sum file seems to be broken: tool="{tool}", exp="{exp}", summary_line="{summary_line}"', + file=sys.stderr) + raise + self.tool = tool + self.exp = exp + except ValueError: + Error('Cannot parse summary line "%s"' % summary_line) + + if self.state not in _VALID_TEST_RESULTS: + Error('Invalid test result %s in "%s" (parsed as "%s")' % ( + self.state, summary_line, self)) + + def __lt__(self, other): + if (self.tool != other.tool): + return self.tool < other.tool + if (self.exp != other.exp): + return self.exp < other.exp + if (self.name != other.name): + return self.name < other.name + return self.ordinal < other.ordinal + + def __hash__(self): + return (hash(self.state) ^ hash(self.tool) ^ hash(self.exp) + ^ hash(self.name) ^ hash(self.description)) + + # Note that we don't include "attrs" in this comparison. This means that + # result entries "FAIL: test" and "flaky | FAIL: test" are considered + # the same. Therefore the ResultSet will preserve only the first occurence. + # In practice this means that flaky entries should preceed expected fails + # entries. + def __eq__(self, other): + return (self.state == other.state and + self.tool == other.tool and + self.exp == other.exp and + self.name == other.name and + self.description == other.description) + + def __ne__(self, other): + return not (self == other) + + def __str__(self): + attrs = '' + if self.attrs: + attrs = '%s | ' % self.attrs + return '%s%s: %s %s' % (attrs, self.state, self.name, self.description) + + def ExpirationDate(self): + # Return a datetime.date object with the expiration date for this + # test result. Return None, if no expiration has been set. + if re.search(r'expire=', self.attrs): + expiration = re.search(r'expire=(\d\d\d\d)(\d\d)(\d\d)', self.attrs) + if not expiration: + Error('Invalid expire= format in "%s". Must be of the form ' + '"expire=YYYYMMDD"' % self) + return datetime.date(int(expiration.group(1)), + int(expiration.group(2)), + int(expiration.group(3))) + return None + + def HasExpired(self): + # Return True if the expiration date of this result has passed. + expiration_date = self.ExpirationDate() + if expiration_date: + return _OPTIONS.expiry_today_date > expiration_date + + +class ResultSet(set): + """Describes a set of DejaGNU test results. + This set can be read in from .sum files or emitted as a manifest. + + Attributes: + current_tool: Name of the current top-level DejaGnu testsuite. + current_exp: Name of the current .exp testsuite file. + testsuites: A set of (tool, exp) tuples representing encountered testsuites. + """ + + def __init__(self): + super().__init__() + self.ResetToolExp() + self.testsuites=set() + + def update(self, other): + super().update(other) + self.testsuites.update(other.testsuites) + + def ResetToolExp(self): + self.current_tool = None + self.current_exp = None + + def MakeTestResult(self, summary_line, ordinal=-1): + return TestResult(summary_line, ordinal, + self.current_tool, self.current_exp) + + def Print(self, outfile=sys.stdout): + current_tool = None + current_exp = None + + for result in sorted(self): + if current_tool != result.tool: + current_tool = result.tool + outfile.write(_TOOL_LINE_FORMAT % current_tool) + if current_exp != result.exp: + current_exp = result.exp + outfile.write(_EXP_LINE_FORMAT % (current_tool, current_exp)) + outfile.write('%s\n' % result) + + outfile.write(_SUMMARY_LINE_FORMAT % 'Results') + + # Check if testsuite of expected_result is present in current results. + # This is used to compare partial test results against a full manifest. + def HasTestsuite(self, expected_result): + return (expected_result.tool, expected_result.exp) in self.testsuites + +def GetMakefileValue(makefile_name, value_name): + if os.path.exists(makefile_name): + makefile = open(makefile_name, encoding='latin-1', mode='r') + for line in makefile: + if line.startswith(value_name): + (_, value) = line.split('=', 1) + value = value.strip() + makefile.close() + return value + makefile.close() + return None + + +def ValidBuildDirectory(builddir): + if (not os.path.exists(builddir) or + not os.path.exists('%s/Makefile' % builddir)): + return False + return True + + +def IsComment(line): + """Return True if line is a comment.""" + return line.startswith('#') + + +def SplitAttributesFromSummaryLine(line): + """Splits off attributes from a summary line, if present.""" + if '|' in line and not _VALID_TEST_RESULTS_REX.match(line): + (attrs, line) = line.split('|', 1) + attrs = attrs.strip() + else: + attrs = '' + line = line.strip() + return (attrs, line) + + +def IsInterestingResult(result_set, line): + """Return True if line is one of the summary lines we care about.""" + (_, line) = SplitAttributesFromSummaryLine(line) + valid_result = bool(_VALID_TEST_RESULTS_REX.match(line)) + + # If there's no tool defined it means that either the results section hasn't + # started yet, or it is already over. + if valid_result and result_set.current_tool is None: + if _OPTIONS.verbosity >= 3: + print(f'WARNING: Result "{line}" found outside sum file boundaries.', + file=sys.stderr) + return False + + return valid_result + + +def IsToolLine(line): + """Return True if line mentions the tool (in DejaGnu terms) for the following tests.""" + return bool(_TOOL_LINE_REX.match(line)) + + +def IsExpLine(line): + """Return True if line mentions the .exp file for the following tests.""" + return bool(_EXP_LINE_REX.match(line)) + + +def IsSummaryLine(line): + """Return True if line starts .sum footer.""" + return bool(_SUMMARY_LINE_REX.match(line)) + + +def IsInclude(line): + """Return True if line is an include of another file.""" + return line.startswith("@include ") + + +def GetIncludeFile(line, includer): + """Extract the name of the include file from line.""" + includer_dir = os.path.dirname(includer) + include_file = line[len("@include "):] + return os.path.join(includer_dir, include_file.strip()) + + +def IsNegativeResult(line): + """Return True if line should be removed from the expected results.""" + return line.startswith("@remove ") + + +def GetNegativeResult(line): + """Extract the name of the negative result from line.""" + line = line[len("@remove "):] + return line.strip() + + +def ParseManifestWorker(result_set, manifest_path): + """Read manifest_path, adding the contents to result_set.""" + if _OPTIONS.verbosity >= 5: + print('Parsing manifest file %s.' % manifest_path) + manifest_file = open(manifest_path, encoding='latin-1', mode='r') + for orig_line in manifest_file: + line = orig_line.strip() + if line == "": + pass + elif IsComment(line): + pass + elif IsNegativeResult(line): + result_set.remove(result_set.MakeTestResult(GetNegativeResult(line))) + elif IsInclude(line): + ParseManifestWorker(result_set, GetIncludeFile(line, manifest_path)) + elif IsInterestingResult(result_set, line): + result = result_set.MakeTestResult(line) + if result.HasExpired(): + # Ignore expired manifest entries. + if _OPTIONS.verbosity >= 4: + print('WARNING: Expected failure "%s" has expired.' % line.strip()) + continue + result_set.add(result) + elif IsExpLine(orig_line): + result_set.current_exp = _EXP_LINE_REX.match(orig_line).groups()[0] + if _OPTIONS.srcpath_regex and _OPTIONS.srcpath_regex != '': + result_set.current_exp = re.sub(_OPTIONS.srcpath_regex, '', + result_set.current_exp) + elif IsToolLine(orig_line): + result_set.current_tool = _TOOL_LINE_REX.match(orig_line).groups()[0] + elif IsSummaryLine(orig_line): + result_set.ResetToolExp() + else: + Error('Unrecognized line in manifest file: %s' % line) + manifest_file.close() + + +def ParseManifest(manifest_path): + """Create a set of TestResult instances from the given manifest file.""" + result_set = ResultSet() + ParseManifestWorker(result_set, manifest_path) + return result_set + + +def ParseSummary(sum_fname): + """Create a set of TestResult instances from the given summary file.""" + result_set = ResultSet() + # ordinal is used when sorting the results so that tests within each + # .exp file are kept sorted. + ordinal=0 + sum_file = open(sum_fname, encoding='latin-1', mode='r') + for line in sum_file: + if IsInterestingResult(result_set, line): + result = result_set.MakeTestResult(line, ordinal) + ordinal += 1 + if result.HasExpired(): + # ??? What is the use-case for this? How "expiry" annotations are + # ??? supposed to be added to .sum results? + # Tests that have expired are not added to the set of expected + # results. If they are still present in the set of actual results, + # they will cause an error to be reported. + if _OPTIONS.verbosity >= 4: + print('WARNING: Expected failure "%s" has expired.' % line.strip()) + continue + result_set.add(result) + elif IsExpLine(line): + result_set.current_exp = _EXP_LINE_REX.match(line).groups()[0] + if _OPTIONS.srcpath_regex and _OPTIONS.srcpath_regex != '': + result_set.current_exp = re.sub(_OPTIONS.srcpath_regex, '', + result_set.current_exp) + result_set.testsuites.add((result_set.current_tool, + result_set.current_exp)) + elif IsToolLine(line): + result_set.current_tool = _TOOL_LINE_REX.match(line).groups()[0] + result_set.current_exp = None + elif IsSummaryLine(line): + result_set.ResetToolExp() + sum_file.close() + return result_set + + +def GetManifest(manifest_path): + """Build a set of expected failures from the manifest file. + + Each entry in the manifest file should have the format understood + by the TestResult constructor. + + If no manifest file exists for this target, it returns an empty set. + """ + if os.path.exists(manifest_path): + return ParseManifest(manifest_path) + else: + return ResultSet() + + +def CollectSumFiles(builddir): + sum_files = [] + for root, dirs, files in os.walk(builddir): + for ignored in ('.svn', '.git'): + if ignored in dirs: + dirs.remove(ignored) + for fname in files: + if fname.endswith('.sum'): + sum_files.append(os.path.join(root, fname)) + return sum_files + + +def GetResults(sum_files, build_results = None): + """Collect all the test results from the given .sum files.""" + if build_results == None: + build_results = ResultSet() + for sum_fname in sum_files: + if _OPTIONS.verbosity >= 3: + print('\t%s' % sum_fname) + build_results.update(ParseSummary(sum_fname)) + return build_results + + +def CompareResults(manifest, actual): + """Compare sets of results and return two lists: + - List of results present in ACTUAL but missing from MANIFEST. + - List of results present in MANIFEST but missing from ACTUAL. + """ + # Collect all the actual results not present in the manifest. + # Results in this set will be reported as errors. + actual_vs_manifest = ResultSet() + for actual_result in actual: + if actual_result not in manifest: + actual_vs_manifest.add(actual_result) + + # Collect all the tests in the manifest that were not found + # in the actual results. + # Results in this set will be reported as warnings (since + # they are expected failures that are not failing anymore). + manifest_vs_actual = ResultSet() + for expected_result in manifest: + # Ignore tests marked flaky. + if 'flaky' in expected_result.attrs: + continue + # We try to support comparing partial results vs full manifest + # (e.g., manifest has failures for gcc, g++, gfortran, but we ran only + # g++ testsuite). To achieve this we record encountered testsuites in + # actual.testsuites set, and then we check it here using HasTestsuite(). + if expected_result not in actual and actual.HasTestsuite(expected_result): + manifest_vs_actual.add(expected_result) + + return actual_vs_manifest, manifest_vs_actual + + +def GetManifestPath(user_provided_must_exist): + """Return the full path to the manifest file.""" + manifest_path = _OPTIONS.manifest + if manifest_path: + if user_provided_must_exist and not os.path.exists(manifest_path): + Error('Manifest does not exist: %s' % manifest_path) + return manifest_path + else: + (srcdir, target) = GetBuildData() + if not srcdir: + Error('Could not determine the location of GCC\'s source tree. ' + 'The Makefile does not contain a definition for "srcdir".') + if not target: + Error('Could not determine the target triplet for this build. ' + 'The Makefile does not contain a definition for "target_alias".') + return _MANIFEST_PATH_PATTERN % (srcdir, _MANIFEST_SUBDIR, target) + + +def GetBuildData(): + if not ValidBuildDirectory(_OPTIONS.build_dir): + # If we have been given a set of results to use, we may + # not be inside a valid GCC build directory. In that case, + # the user must provide both a manifest file and a set + # of results to check against it. + if not _OPTIONS.results or not _OPTIONS.manifest: + Error('%s is not a valid GCC top level build directory. ' + 'You must use --manifest and --results to do the validation.' % + _OPTIONS.build_dir) + else: + return None, None + srcdir = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'srcdir =') + target = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'target_alias=') + if _OPTIONS.verbosity >= 3: + print('Source directory: %s' % srcdir) + print('Build target: %s' % target) + return srcdir, target + + +def PrintSummary(summary): + summary.Print() + +def GetSumFiles(results, build_dir): + if not results: + if _OPTIONS.verbosity >= 3: + print('Getting actual results from build directory %s' % build_dir) + sum_files = CollectSumFiles(build_dir) + else: + if _OPTIONS.verbosity >= 3: + print('Getting actual results from user-provided results') + sum_files = results.split() + return sum_files + + +def PerformComparison(expected, actual): + actual_vs_expected, expected_vs_actual = CompareResults(expected, actual) + + if _OPTIONS.inverse_match: + # Switch results if inverse comparison is requested. + # This is useful in detecting flaky tests that FAILed in expected set, + # but PASSed in actual set. + actual_vs_expected, expected_vs_actual \ + = expected_vs_actual, actual_vs_expected + + tests_ok = True + if len(actual_vs_expected) > 0: + if _OPTIONS.verbosity >= 3: + print('\n\nUnexpected results in this build (new failures)') + if _OPTIONS.verbosity >= 1: + PrintSummary(actual_vs_expected) + tests_ok = False + + if _OPTIONS.verbosity >= 2 and len(expected_vs_actual) > 0: + print('\n\nExpected results not present in this build (fixed tests)' + '\n\nNOTE: This is not a failure. It just means that these ' + 'tests were expected\nto fail, but either they worked in ' + 'this configuration or they were not\npresent at all.\n') + PrintSummary(expected_vs_actual) + + if tests_ok and _OPTIONS.verbosity >= 3: + print('\nSUCCESS: No unexpected failures.') + + return tests_ok + + +def CheckExpectedResults(): + manifest_path = GetManifestPath(True) + if _OPTIONS.verbosity >= 3: + print('Manifest: %s' % manifest_path) + manifest = GetManifest(manifest_path) + sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir) + actual = GetResults(sum_files) + + if _OPTIONS.verbosity >= 5: + print('\n\nTests expected to fail') + PrintSummary(manifest) + print('\n\nActual test results') + PrintSummary(actual) + + return PerformComparison(manifest, actual) + + +def ProduceManifest(): + manifest_path = GetManifestPath(False) + if _OPTIONS.verbosity >= 3: + print('Manifest: %s' % manifest_path) + if os.path.exists(manifest_path) and not _OPTIONS.force: + Error('Manifest file %s already exists.\nUse --force to overwrite.' % + manifest_path) + + sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir) + actual = GetResults(sum_files) + manifest_file = open(manifest_path, encoding='latin-1', mode='w') + actual.Print(manifest_file) + actual.Print() + manifest_file.close() + + return True + + +def CompareBuilds(): + sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir) + actual = GetResults(sum_files) + + clean = ResultSet() + + if _OPTIONS.manifest: + manifest_path = GetManifestPath(True) + if _OPTIONS.verbosity >= 3: + print('Manifest: %s' % manifest_path) + clean = GetManifest(manifest_path) + + clean_sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.clean_build) + clean = GetResults(clean_sum_files, clean) + + return PerformComparison(clean, actual) + + +def Main(argv): + parser = optparse.OptionParser(usage=__doc__) + + # Keep the following list sorted by option name. + parser.add_option('--build_dir', action='store', type='string', + dest='build_dir', default='.', + help='Build directory to check (default = .)') + parser.add_option('--clean_build', action='store', type='string', + dest='clean_build', default=None, + help='Compare test results from this build against ' + 'those of another (clean) build. Use this option ' + 'when comparing the test results of your patch versus ' + 'the test results of a clean build without your patch. ' + 'You must provide the path to the top directory of your ' + 'clean build.') + parser.add_option('--force', action='store_true', dest='force', + default=False, help='When used with --produce_manifest, ' + 'it will overwrite an existing manifest file ' + '(default = False)') + parser.add_option('--expiry_date', action='store', + dest='expiry_today_date', default=None, + help='Use provided date YYYYMMDD to decide whether ' + 'manifest entries with expiry settings have expired ' + 'or not. (default = Use today date)') + parser.add_option('--srcpath', action='store', type='string', + dest='srcpath_regex', default='[^ ]+/testsuite/', + help='Remove provided path (can be a regex) from ' + 'the result entries. This is useful to remove ' + 'occasional filesystem path from the results. ' + '(default = "[^ ]+/testsuite/")') + parser.add_option('--inverse_match', action='store_true', + dest='inverse_match', default=False, + help='Inverse result sets in comparison. ' + 'Output unexpected passes as unexpected failures and ' + 'unexpected failures as unexpected passes. ' + 'This is used to catch FAIL->PASS flaky tests. ' + '(default = False)') + parser.add_option('--manifest', action='store', type='string', + dest='manifest', default=None, + help='Name of the manifest file to use (default = ' + 'taken from ' + 'contrib/testsuite-managment/<target_alias>.xfail)') + parser.add_option('--produce_manifest', action='store_true', + dest='produce_manifest', default=False, + help='Produce the manifest for the current ' + 'build (default = False)') + parser.add_option('--results', action='store', type='string', + dest='results', default=None, help='Space-separated list ' + 'of .sum files with the testing results to check. The ' + 'only content needed from these files are the lines ' + 'starting with FAIL, XPASS or UNRESOLVED (default = ' + '.sum files collected from the build directory).') + parser.add_option('--verbosity', action='store', dest='verbosity', + type='int', default=3, help='Verbosity level ' + '(default = 3). Level 0: only error output, this is ' + 'useful in scripting when only the exit code is used. ' + 'Level 1: output unexpected failures. ' + 'Level 2: output unexpected passes. ' + 'Level 3: output helpful information. ' + 'Level 4: output notification on expired entries. ' + 'Level 5: output debug information.') + global _OPTIONS + (_OPTIONS, _) = parser.parse_args(argv[1:]) + + # Set "today" date to compare expiration entries against. + # Setting expiration date into the future allows re-detection of flaky + # tests and creating fresh entries for them before the current flaky entries + # expire. + if _OPTIONS.expiry_today_date: + today_date = re.search(r'(\d\d\d\d)(\d\d)(\d\d)', + _OPTIONS.expiry_today_date) + if not today_date: + Error('Invalid --expiry_today_date format "%s". Must be of the form ' + '"expire=YYYYMMDD"' % _OPTIONS.expiry_today_date) + _OPTIONS.expiry_today_date=datetime.date(int(today_date.group(1)), + int(today_date.group(2)), + int(today_date.group(3))) + else: + _OPTIONS.expiry_today_date = datetime.date.today() + + if _OPTIONS.produce_manifest: + retval = ProduceManifest() + elif _OPTIONS.clean_build: + retval = CompareBuilds() + else: + retval = CheckExpectedResults() + + if retval: + return 0 + else: + return 2 + + +if __name__ == '__main__': + retval = Main(sys.argv) + sys.exit(retval) diff --git a/scripts/testsuite-management/x86_64-unknown-linux-gnu.xfail b/scripts/testsuite-management/x86_64-unknown-linux-gnu.xfail new file mode 100644 index 0000000..fdb8268 --- /dev/null +++ b/scripts/testsuite-management/x86_64-unknown-linux-gnu.xfail @@ -0,0 +1,109 @@ +FAIL: g++.dg/guality/pr55665.C -O2 line 23 p == 40 +FAIL: g++.dg/guality/pr55665.C -O3 -fomit-frame-pointer line 23 p == 40 +FAIL: g++.dg/guality/pr55665.C -O3 -g line 23 p == 40 +XPASS: gcc.dg/guality/example.c -O0 execution test +XPASS: gcc.dg/guality/example.c -O2 execution test +XPASS: gcc.dg/guality/example.c -O2 -flto -fno-use-linker-plugin -flto-partition=none execution test +XPASS: gcc.dg/guality/guality.c -O0 execution test +XPASS: gcc.dg/guality/guality.c -O1 execution test +XPASS: gcc.dg/guality/guality.c -O2 execution test +XPASS: gcc.dg/guality/guality.c -O3 -fomit-frame-pointer execution test +XPASS: gcc.dg/guality/guality.c -O3 -g execution test +XPASS: gcc.dg/guality/guality.c -Os execution test +XPASS: gcc.dg/guality/guality.c -O2 -flto -fno-use-linker-plugin -flto-partition=none execution test +XPASS: gcc.dg/guality/guality.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects execution test +XPASS: gcc.dg/guality/inline-params.c -O2 execution test +XPASS: gcc.dg/guality/inline-params.c -O3 -fomit-frame-pointer execution test +XPASS: gcc.dg/guality/inline-params.c -O3 -g execution test +XPASS: gcc.dg/guality/inline-params.c -Os execution test +XPASS: gcc.dg/guality/inline-params.c -O2 -flto -fno-use-linker-plugin -flto-partition=none execution test +XPASS: gcc.dg/guality/pr41353-1.c -O0 line 28 j == 28 + 37 +XPASS: gcc.dg/guality/pr41353-1.c -O1 line 28 j == 28 + 37 +XPASS: gcc.dg/guality/pr41353-1.c -O2 line 28 j == 28 + 37 +XPASS: gcc.dg/guality/pr41353-1.c -O3 -fomit-frame-pointer line 28 j == 28 + 37 +XPASS: gcc.dg/guality/pr41353-1.c -O3 -g line 28 j == 28 + 37 +XPASS: gcc.dg/guality/pr41353-1.c -Os line 28 j == 28 + 37 +XPASS: gcc.dg/guality/pr41353-1.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 28 j == 28 + 37 +XPASS: gcc.dg/guality/pr41353-1.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 28 j == 28 + 37 +XPASS: gcc.dg/guality/pr41447-1.c -O0 execution test +XPASS: gcc.dg/guality/pr41447-1.c -O1 execution test +XPASS: gcc.dg/guality/pr41447-1.c -O2 execution test +XPASS: gcc.dg/guality/pr41447-1.c -O3 -fomit-frame-pointer execution test +XPASS: gcc.dg/guality/pr41447-1.c -O3 -g execution test +XPASS: gcc.dg/guality/pr41447-1.c -Os execution test +XPASS: gcc.dg/guality/pr41447-1.c -O2 -flto -fno-use-linker-plugin -flto-partition=none execution test +XPASS: gcc.dg/guality/pr41447-1.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects execution test +XPASS: gcc.dg/guality/pr41616-1.c -O0 execution test +XPASS: gcc.dg/guality/pr41616-1.c -O1 execution test +XPASS: gcc.dg/guality/pr41616-1.c -O2 execution test +XPASS: gcc.dg/guality/pr41616-1.c -O3 -fomit-frame-pointer execution test +XPASS: gcc.dg/guality/pr41616-1.c -O3 -g execution test +XPASS: gcc.dg/guality/pr41616-1.c -Os execution test +XPASS: gcc.dg/guality/pr41616-1.c -O2 -flto -fno-use-linker-plugin -flto-partition=none execution test +FAIL: gcc.dg/guality/pr43051-1.c -O3 -fomit-frame-pointer -funroll-loops line 39 c == &a[0] +FAIL: gcc.dg/guality/pr43051-1.c -O3 -fomit-frame-pointer -funroll-all-loops -finline-functions line 39 c == &a[0] +FAIL: gcc.dg/guality/pr48437.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 14 i == 0 +FAIL: gcc.dg/guality/pr48437.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 14 i == 0 +FAIL: gcc.dg/guality/pr54200.c -Os line 20 z == 3 +FAIL: gcc.dg/guality/pr54519-1.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 20 y == 25 +FAIL: gcc.dg/guality/pr54519-1.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 20 z == 6 +FAIL: gcc.dg/guality/pr54519-1.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 23 y == 117 +FAIL: gcc.dg/guality/pr54519-1.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 23 z == 8 +FAIL: gcc.dg/guality/pr54519-1.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 20 y == 25 +FAIL: gcc.dg/guality/pr54519-1.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 20 z == 6 +FAIL: gcc.dg/guality/pr54519-2.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 17 y == 25 +FAIL: gcc.dg/guality/pr54519-2.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 17 y == 25 +FAIL: gcc.dg/guality/pr54519-3.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 20 y == 25 +FAIL: gcc.dg/guality/pr54519-3.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 20 z == 6 +FAIL: gcc.dg/guality/pr54519-3.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 23 y == 117 +FAIL: gcc.dg/guality/pr54519-3.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 23 z == 8 +FAIL: gcc.dg/guality/pr54519-3.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 20 y == 25 +FAIL: gcc.dg/guality/pr54519-3.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 20 z == 6 +FAIL: gcc.dg/guality/pr54519-3.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 23 y == 117 +FAIL: gcc.dg/guality/pr54519-3.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 23 z == 8 +FAIL: gcc.dg/guality/pr54519-4.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 17 y == 25 +FAIL: gcc.dg/guality/pr54519-4.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 17 y == 25 +FAIL: gcc.dg/guality/pr54519-5.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 17 y == 25 +FAIL: gcc.dg/guality/pr54519-5.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 17 y == 25 +FAIL: gcc.dg/guality/pr54693-2.c -Os line 21 x == 10 - i +FAIL: gcc.dg/guality/pr54693-2.c -Os line 21 y == 20 - 2 * i +FAIL: gcc.dg/guality/vla-1.c -O0 line 17 sizeof (a) == 6 +FAIL: gcc.dg/guality/vla-1.c -O0 line 24 sizeof (a) == 17 * sizeof (short) +FAIL: gcc.dg/guality/vla-1.c -O1 line 17 sizeof (a) == 6 +FAIL: gcc.dg/guality/vla-1.c -O1 line 24 sizeof (a) == 17 * sizeof (short) +FAIL: gcc.dg/guality/vla-1.c -O2 line 17 sizeof (a) == 6 +FAIL: gcc.dg/guality/vla-1.c -O2 line 24 sizeof (a) == 17 * sizeof (short) +FAIL: gcc.dg/guality/vla-1.c -O3 -fomit-frame-pointer line 17 sizeof (a) == 6 +FAIL: gcc.dg/guality/vla-1.c -O3 -fomit-frame-pointer line 24 sizeof (a) == 17 * sizeof (short) +FAIL: gcc.dg/guality/vla-1.c -O3 -g line 17 sizeof (a) == 6 +FAIL: gcc.dg/guality/vla-1.c -O3 -g line 24 sizeof (a) == 17 * sizeof (short) +FAIL: gcc.dg/guality/vla-1.c -Os line 17 sizeof (a) == 6 +FAIL: gcc.dg/guality/vla-1.c -Os line 24 sizeof (a) == 17 * sizeof (short) +FAIL: gcc.dg/guality/vla-1.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 17 sizeof (a) == 6 +FAIL: gcc.dg/guality/vla-1.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 24 sizeof (a) == 17 * sizeof (short) +FAIL: gcc.dg/guality/vla-1.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 17 sizeof (a) == 6 +FAIL: gcc.dg/guality/vla-1.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 24 sizeof (a) == 17 * sizeof (short) +FAIL: gcc.dg/guality/vla-2.c -O0 line 16 sizeof (a) == 5 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O0 line 25 sizeof (a) == 6 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O1 line 16 sizeof (a) == 5 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O1 line 25 sizeof (a) == 6 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O2 line 16 sizeof (a) == 5 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O2 line 25 sizeof (a) == 6 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O3 -fomit-frame-pointer line 16 sizeof (a) == 5 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O3 -fomit-frame-pointer line 25 sizeof (a) == 6 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O3 -g line 16 sizeof (a) == 5 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O3 -g line 25 sizeof (a) == 6 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -Os line 16 sizeof (a) == 5 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -Os line 25 sizeof (a) == 6 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 16 sizeof (a) == 5 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O2 -flto -fno-use-linker-plugin -flto-partition=none line 25 sizeof (a) == 6 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 16 sizeof (a) == 5 * sizeof (int) +FAIL: gcc.dg/guality/vla-2.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects line 25 sizeof (a) == 6 * sizeof (int) +FAIL: gcc.dg/tree-ssa/slsr-39.c scan-tree-dump-times slsr "MEM" 4 +FAIL: libmudflap.c++/pass41-frag.cxx ( -O) execution test +FAIL: libmudflap.c++/pass41-frag.cxx (-O2) execution test +FAIL: libmudflap.c++/pass41-frag.cxx (-O3) execution test +FAIL: libmudflap.c++/pass55-frag.cxx (-O3) execution test +FAIL: sourcelocation output - source compiled test +FAIL: sourcelocation -findirect-dispatch output - source compiled test +FAIL: sourcelocation -O3 output - source compiled test
