Yea, that actually came up on my radar too, but when I tried to do more things at once it started turning into a rats nest. Refactoring Python is kind of hard so I want to make the changes as small as possible that if anything breaks, bisecting will narrow the problem down as much as possible.
On Mon, Dec 7, 2015 at 3:35 PM Todd Fiala <todd.fi...@gmail.com> wrote: > This is okay, but I was planning on breaking that out into multiple > files. Right now the test_results module also has the EventBuilder in it, > which is not a results formatter. But we can break that out separately. > > On Mon, Dec 7, 2015 at 1:23 PM, Zachary Turner via lldb-commits < > lldb-commits@lists.llvm.org> wrote: > >> Author: zturner >> Date: Mon Dec 7 15:23:41 2015 >> New Revision: 254946 >> >> URL: http://llvm.org/viewvc/llvm-project?rev=254946&view=rev >> Log: >> Rename test_results.py to result_formatter.py. >> >> There is already a class called LLDBTestResults which I would like >> to move into a separate file, but the most appropriate filename >> was taken. >> >> Added: >> lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py >> - copied, changed from r254944, >> lldb/trunk/packages/Python/lldbsuite/test/test_results.py >> Removed: >> lldb/trunk/packages/Python/lldbsuite/test/test_results.py >> Modified: >> lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py >> lldb/trunk/packages/Python/lldbsuite/test/curses_results.py >> lldb/trunk/packages/Python/lldbsuite/test/dosep.py >> lldb/trunk/packages/Python/lldbsuite/test/dotest.py >> >> Modified: >> lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py >> URL: >> http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py?rev=254946&r1=254945&r2=254946&view=diff >> >> ============================================================================== >> --- lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py >> (original) >> +++ lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py >> Mon Dec 7 15:23:41 2015 >> @@ -13,10 +13,11 @@ from __future__ import print_function >> import os >> >> # Our imports >> -from . import test_results >> +from . import result_formatter >> import lldbsuite >> >> -class BasicResultsFormatter(test_results.ResultsFormatter): >> + >> +class BasicResultsFormatter(result_formatter.ResultsFormatter): >> """Provides basic test result output.""" >> @classmethod >> def arg_parser(cls): >> @@ -240,16 +241,16 @@ class BasicResultsFormatter(test_results >> # Output each of the test result entries. >> categories = [ >> # result id, printed name, print matching tests?, detail >> label >> - [test_results.EventBuilder.STATUS_SUCCESS, >> + [result_formatter.EventBuilder.STATUS_SUCCESS, >> "Success", False, None], >> - [test_results.EventBuilder.STATUS_EXPECTED_FAILURE, >> + [result_formatter.EventBuilder.STATUS_EXPECTED_FAILURE, >> "Expected Failure", False, None], >> - [test_results.EventBuilder.STATUS_FAILURE, >> + [result_formatter.EventBuilder.STATUS_FAILURE, >> "Failure", True, "FAIL"], >> - [test_results.EventBuilder.STATUS_ERROR, "Error", True, >> "ERROR"], >> - [test_results.EventBuilder.STATUS_UNEXPECTED_SUCCESS, >> + [result_formatter.EventBuilder.STATUS_ERROR, "Error", True, >> "ERROR"], >> + [result_formatter.EventBuilder.STATUS_UNEXPECTED_SUCCESS, >> "Unexpected Success", True, "UNEXPECTED SUCCESS"], >> - [test_results.EventBuilder.STATUS_SKIP, "Skip", False, None]] >> + [result_formatter.EventBuilder.STATUS_SKIP, "Skip", False, >> None]] >> >> # Partition all the events by test result status >> result_events_by_status = self._partition_results_by_status( >> >> Modified: lldb/trunk/packages/Python/lldbsuite/test/curses_results.py >> URL: >> http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/curses_results.py?rev=254946&r1=254945&r2=254946&view=diff >> >> ============================================================================== >> --- lldb/trunk/packages/Python/lldbsuite/test/curses_results.py (original) >> +++ lldb/trunk/packages/Python/lldbsuite/test/curses_results.py Mon Dec >> 7 15:23:41 2015 >> @@ -23,11 +23,11 @@ import time >> >> # LLDB modules >> from . import lldbcurses >> -from . import test_results >> -from .test_results import EventBuilder >> +from . import result_formatter >> +from .result_formatter import EventBuilder >> >> >> -class Curses(test_results.ResultsFormatter): >> +class Curses(result_formatter.ResultsFormatter): >> """Receives live results from tests that are running and reports >> them to the terminal in a curses GUI""" >> >> def __init__(self, out_file, options): >> >> Modified: lldb/trunk/packages/Python/lldbsuite/test/dosep.py >> URL: >> http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/dosep.py?rev=254946&r1=254945&r2=254946&view=diff >> >> ============================================================================== >> --- lldb/trunk/packages/Python/lldbsuite/test/dosep.py (original) >> +++ lldb/trunk/packages/Python/lldbsuite/test/dosep.py Mon Dec 7 >> 15:23:41 2015 >> @@ -53,7 +53,7 @@ import lldbsuite.support.seven as seven >> >> from . import dotest_channels >> from . import dotest_args >> -from . import test_results >> +from . import result_formatter >> >> # Todo: Convert this folder layout to be relative-import friendly and >> don't hack up >> # sys.path like this >> @@ -1429,9 +1429,9 @@ def main(print_details_on_success, num_t >> # Figure out exit code by count of test result types. >> issue_count = ( >> results_formatter.counts_by_test_result_status( >> - test_results.EventBuilder.STATUS_ERROR) + >> + result_formatter.EventBuilder.STATUS_ERROR) + >> results_formatter.counts_by_test_result_status( >> - test_results.EventBuilder.STATUS_FAILURE) + >> + result_formatter.EventBuilder.STATUS_FAILURE) + >> timeout_count) >> # Return with appropriate result code >> if issue_count > 0: >> >> Modified: lldb/trunk/packages/Python/lldbsuite/test/dotest.py >> URL: >> http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/dotest.py?rev=254946&r1=254945&r2=254946&view=diff >> >> ============================================================================== >> --- lldb/trunk/packages/Python/lldbsuite/test/dotest.py (original) >> +++ lldb/trunk/packages/Python/lldbsuite/test/dotest.py Mon Dec 7 >> 15:23:41 2015 >> @@ -43,8 +43,8 @@ import lldbsuite >> from . import dotest_args >> from . import lldbtest_config >> from . import test_categories >> -from . import test_results >> -from .test_results import EventBuilder >> +from . import result_formatter >> +from .result_formatter import EventBuilder >> from ..support import seven >> >> def is_exe(fpath): >> @@ -795,7 +795,7 @@ def parseOptionsAndInitTestdirs(): >> # Tell the event builder to create all events with these >> # key/val pairs in them. >> if len(entries) > 0: >> - test_results.EventBuilder.add_entries_to_all_events(entries) >> + >> result_formatter.EventBuilder.add_entries_to_all_events(entries) >> >> # Gather all the dirs passed on the command line. >> if len(args.args) > 0: >> @@ -930,13 +930,13 @@ def setupTestResults(): >> else: >> results_file_object = open(results_filename, "w") >> cleanup_func = results_file_object.close >> - default_formatter_name = >> "lldbsuite.test.test_results.XunitFormatter" >> + default_formatter_name = >> "lldbsuite.test.result_formatter.XunitFormatter" >> elif results_port: >> # Connect to the specified localhost port. >> results_file_object, cleanup_func = createSocketToLocalPort( >> results_port) >> default_formatter_name = ( >> - "lldbsuite.test.test_results.RawPickledFormatter") >> + "lldbsuite.test.result_formatter.RawPickledFormatter") >> >> # If we have a results formatter name specified and we didn't specify >> # a results file, we should use stdout. >> >> Copied: lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py >> (from r254944, lldb/trunk/packages/Python/lldbsuite/test/test_results.py) >> URL: >> http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py?p2=lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py&p1=lldb/trunk/packages/Python/lldbsuite/test/test_results.py&r1=254944&r2=254946&rev=254946&view=diff >> >> ============================================================================== >> (empty) >> >> Removed: lldb/trunk/packages/Python/lldbsuite/test/test_results.py >> URL: >> http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/test_results.py?rev=254945&view=auto >> >> ============================================================================== >> --- lldb/trunk/packages/Python/lldbsuite/test/test_results.py (original) >> +++ lldb/trunk/packages/Python/lldbsuite/test/test_results.py (removed) >> @@ -1,1042 +0,0 @@ >> -""" >> - The LLVM Compiler Infrastructure >> - >> -This file is distributed under the University of Illinois Open Source >> -License. See LICENSE.TXT for details. >> - >> -Provides classes used by the test results reporting infrastructure >> -within the LLDB test suite. >> -""" >> - >> -from __future__ import print_function >> -from __future__ import absolute_import >> - >> -# System modules >> -import argparse >> -import inspect >> -import os >> -import pprint >> -import re >> -import sys >> -import threading >> -import time >> -import traceback >> -import xml.sax.saxutils >> - >> -# Third-party modules >> -import six >> -from six.moves import cPickle >> - >> -# LLDB modules >> - >> - >> -class EventBuilder(object): >> - """Helper class to build test result event dictionaries.""" >> - >> - BASE_DICTIONARY = None >> - >> - # Test Status Tags >> - STATUS_SUCCESS = "success" >> - STATUS_FAILURE = "failure" >> - STATUS_EXPECTED_FAILURE = "expected_failure" >> - STATUS_UNEXPECTED_SUCCESS = "unexpected_success" >> - STATUS_SKIP = "skip" >> - STATUS_ERROR = "error" >> - >> - @staticmethod >> - def _get_test_name_info(test): >> - """Returns (test-class-name, test-method-name) from a test case >> instance. >> - >> - @param test a unittest.TestCase instance. >> - >> - @return tuple containing (test class name, test method name) >> - """ >> - test_class_components = test.id().split(".") >> - test_class_name = ".".join(test_class_components[:-1]) >> - test_name = test_class_components[-1] >> - return (test_class_name, test_name) >> - >> - @staticmethod >> - def bare_event(event_type): >> - """Creates an event with default additions, event type and >> timestamp. >> - >> - @param event_type the value set for the "event" key, used >> - to distinguish events. >> - >> - @returns an event dictionary with all default additions, the >> "event" >> - key set to the passed in event_type, and the event_time value >> set to >> - time.time(). >> - """ >> - if EventBuilder.BASE_DICTIONARY is not None: >> - # Start with a copy of the "always include" entries. >> - event = dict(EventBuilder.BASE_DICTIONARY) >> - else: >> - event = {} >> - >> - event.update({ >> - "event": event_type, >> - "event_time": time.time() >> - }) >> - return event >> - >> - @staticmethod >> - def _event_dictionary_common(test, event_type): >> - """Returns an event dictionary setup with values for the given >> event type. >> - >> - @param test the unittest.TestCase instance >> - >> - @param event_type the name of the event type (string). >> - >> - @return event dictionary with common event fields set. >> - """ >> - test_class_name, test_name = >> EventBuilder._get_test_name_info(test) >> - >> - event = EventBuilder.bare_event(event_type) >> - event.update({ >> - "test_class": test_class_name, >> - "test_name": test_name, >> - "test_filename": inspect.getfile(test.__class__) >> - }) >> - return event >> - >> - @staticmethod >> - def _error_tuple_class(error_tuple): >> - """Returns the unittest error tuple's error class as a string. >> - >> - @param error_tuple the error tuple provided by the test >> framework. >> - >> - @return the error type (typically an exception) raised by the >> - test framework. >> - """ >> - type_var = error_tuple[0] >> - module = inspect.getmodule(type_var) >> - if module: >> - return "{}.{}".format(module.__name__, type_var.__name__) >> - else: >> - return type_var.__name__ >> - >> - @staticmethod >> - def _error_tuple_message(error_tuple): >> - """Returns the unittest error tuple's error message. >> - >> - @param error_tuple the error tuple provided by the test >> framework. >> - >> - @return the error message provided by the test framework. >> - """ >> - return str(error_tuple[1]) >> - >> - @staticmethod >> - def _error_tuple_traceback(error_tuple): >> - """Returns the unittest error tuple's error message. >> - >> - @param error_tuple the error tuple provided by the test >> framework. >> - >> - @return the error message provided by the test framework. >> - """ >> - return error_tuple[2] >> - >> - @staticmethod >> - def _event_dictionary_test_result(test, status): >> - """Returns an event dictionary with common test result fields >> set. >> - >> - @param test a unittest.TestCase instance. >> - >> - @param status the status/result of the test >> - (e.g. "success", "failure", etc.) >> - >> - @return the event dictionary >> - """ >> - event = EventBuilder._event_dictionary_common(test, >> "test_result") >> - event["status"] = status >> - return event >> - >> - @staticmethod >> - def _event_dictionary_issue(test, status, error_tuple): >> - """Returns an event dictionary with common issue-containing test >> result >> - fields set. >> - >> - @param test a unittest.TestCase instance. >> - >> - @param status the status/result of the test >> - (e.g. "success", "failure", etc.) >> - >> - @param error_tuple the error tuple as reported by the test >> runner. >> - This is of the form (type<error>, error). >> - >> - @return the event dictionary >> - """ >> - event = EventBuilder._event_dictionary_test_result(test, status) >> - event["issue_class"] = >> EventBuilder._error_tuple_class(error_tuple) >> - event["issue_message"] = >> EventBuilder._error_tuple_message(error_tuple) >> - backtrace = EventBuilder._error_tuple_traceback(error_tuple) >> - if backtrace is not None: >> - event["issue_backtrace"] = traceback.format_tb(backtrace) >> - return event >> - >> - @staticmethod >> - def event_for_start(test): >> - """Returns an event dictionary for the test start event. >> - >> - @param test a unittest.TestCase instance. >> - >> - @return the event dictionary >> - """ >> - return EventBuilder._event_dictionary_common(test, "test_start") >> - >> - @staticmethod >> - def event_for_success(test): >> - """Returns an event dictionary for a successful test. >> - >> - @param test a unittest.TestCase instance. >> - >> - @return the event dictionary >> - """ >> - return EventBuilder._event_dictionary_test_result( >> - test, EventBuilder.STATUS_SUCCESS) >> - >> - @staticmethod >> - def event_for_unexpected_success(test, bugnumber): >> - """Returns an event dictionary for a test that succeeded but was >> - expected to fail. >> - >> - @param test a unittest.TestCase instance. >> - >> - @param bugnumber the issue identifier for the bug tracking the >> - fix request for the test expected to fail (but is in fact >> - passing here). >> - >> - @return the event dictionary >> - >> - """ >> - event = EventBuilder._event_dictionary_test_result( >> - test, EventBuilder.STATUS_UNEXPECTED_SUCCESS) >> - if bugnumber: >> - event["bugnumber"] = str(bugnumber) >> - return event >> - >> - @staticmethod >> - def event_for_failure(test, error_tuple): >> - """Returns an event dictionary for a test that failed. >> - >> - @param test a unittest.TestCase instance. >> - >> - @param error_tuple the error tuple as reported by the test >> runner. >> - This is of the form (type<error>, error). >> - >> - @return the event dictionary >> - """ >> - return EventBuilder._event_dictionary_issue( >> - test, EventBuilder.STATUS_FAILURE, error_tuple) >> - >> - @staticmethod >> - def event_for_expected_failure(test, error_tuple, bugnumber): >> - """Returns an event dictionary for a test that failed as >> expected. >> - >> - @param test a unittest.TestCase instance. >> - >> - @param error_tuple the error tuple as reported by the test >> runner. >> - This is of the form (type<error>, error). >> - >> - @param bugnumber the issue identifier for the bug tracking the >> - fix request for the test expected to fail. >> - >> - @return the event dictionary >> - >> - """ >> - event = EventBuilder._event_dictionary_issue( >> - test, EventBuilder.STATUS_EXPECTED_FAILURE, error_tuple) >> - if bugnumber: >> - event["bugnumber"] = str(bugnumber) >> - return event >> - >> - @staticmethod >> - def event_for_skip(test, reason): >> - """Returns an event dictionary for a test that was skipped. >> - >> - @param test a unittest.TestCase instance. >> - >> - @param reason the reason why the test is being skipped. >> - >> - @return the event dictionary >> - """ >> - event = EventBuilder._event_dictionary_test_result( >> - test, EventBuilder.STATUS_SKIP) >> - event["skip_reason"] = reason >> - return event >> - >> - @staticmethod >> - def event_for_error(test, error_tuple): >> - """Returns an event dictionary for a test that hit a test >> execution error. >> - >> - @param test a unittest.TestCase instance. >> - >> - @param error_tuple the error tuple as reported by the test >> runner. >> - This is of the form (type<error>, error). >> - >> - @return the event dictionary >> - """ >> - return EventBuilder._event_dictionary_issue( >> - test, EventBuilder.STATUS_ERROR, error_tuple) >> - >> - @staticmethod >> - def event_for_cleanup_error(test, error_tuple): >> - """Returns an event dictionary for a test that hit a test >> execution error >> - during the test cleanup phase. >> - >> - @param test a unittest.TestCase instance. >> - >> - @param error_tuple the error tuple as reported by the test >> runner. >> - This is of the form (type<error>, error). >> - >> - @return the event dictionary >> - """ >> - event = EventBuilder._event_dictionary_issue( >> - test, EventBuilder.STATUS_ERROR, error_tuple) >> - event["issue_phase"] = "cleanup" >> - return event >> - >> - @staticmethod >> - def add_entries_to_all_events(entries_dict): >> - """Specifies a dictionary of entries to add to all test events. >> - >> - This provides a mechanism for, say, a parallel test runner to >> - indicate to each inferior dotest.py that it should add a >> - worker index to each. >> - >> - Calling this method replaces all previous entries added >> - by a prior call to this. >> - >> - Event build methods will overwrite any entries that collide. >> - Thus, the passed in dictionary is the base, which gets merged >> - over by event building when keys collide. >> - >> - @param entries_dict a dictionary containing key and value >> - pairs that should be merged into all events created by the >> - event generator. May be None to clear out any extra entries. >> - """ >> - EventBuilder.BASE_DICTIONARY = dict(entries_dict) >> - >> - >> -class ResultsFormatter(object): >> - >> - """Provides interface to formatting test results out to a file-like >> object. >> - >> - This class allows the LLDB test framework's raw test-realted >> - events to be processed and formatted in any manner desired. >> - Test events are represented by python dictionaries, formatted >> - as in the EventBuilder class above. >> - >> - ResultFormatter instances are given a file-like object in which >> - to write their results. >> - >> - ResultFormatter lifetime looks like the following: >> - >> - # The result formatter is created. >> - # The argparse options dictionary is generated from calling >> - # the SomeResultFormatter.arg_parser() with the options data >> - # passed to dotest.py via the "--results-formatter-options" >> - # argument. See the help on that for syntactic requirements >> - # on getting that parsed correctly. >> - formatter = SomeResultFormatter(file_like_object, >> argpared_options_dict) >> - >> - # Single call to session start, before parsing any events. >> - formatter.begin_session() >> - >> - formatter.handle_event({"event":"initialize",...}) >> - >> - # Zero or more calls specified for events recorded during the test >> session. >> - # The parallel test runner manages getting results from all the >> inferior >> - # dotest processes, so from a new format perspective, don't worry >> about >> - # that. The formatter will be presented with a single stream of >> events >> - # sandwiched between a single begin_session()/end_session() pair in >> the >> - # parallel test runner process/thread. >> - for event in zero_or_more_test_events(): >> - formatter.handle_event(event) >> - >> - # Single call to terminate/wrap-up. Formatters that need all the >> - # data before they can print a correct result (e.g. xUnit/JUnit), >> - # this is where the final report can be generated. >> - formatter.handle_event({"event":"terminate",...}) >> - >> - It is not the formatter's responsibility to close the >> file_like_object. >> - (i.e. do not close it). >> - >> - The lldb test framework passes these test events in real time, so >> they >> - arrive as they come in. >> - >> - In the case of the parallel test runner, the dotest inferiors >> - add a 'pid' field to the dictionary that indicates which inferior >> - pid generated the event. >> - >> - Note more events may be added in the future to support richer test >> - reporting functionality. One example: creating a true flaky test >> - result category so that unexpected successes really mean the test >> - is marked incorrectly (either should be marked flaky, or is indeed >> - passing consistently now and should have the xfail marker >> - removed). In this case, a flaky_success and flaky_fail event >> - likely will be added to capture these and support reporting things >> - like percentages of flaky test passing so we can see if we're >> - making some things worse/better with regards to failure rates. >> - >> - Another example: announcing all the test methods that are planned >> - to be run, so we can better support redo operations of various kinds >> - (redo all non-run tests, redo non-run tests except the one that >> - was running [perhaps crashed], etc.) >> - >> - Implementers are expected to override all the public methods >> - provided in this class. See each method's docstring to see >> - expectations about when the call should be chained. >> - >> - """ >> - @classmethod >> - def arg_parser(cls): >> - """@return arg parser used to parse formatter-specific >> options.""" >> - parser = argparse.ArgumentParser( >> - description='{} options'.format(cls.__name__), >> - usage=('dotest.py --results-formatter-options=' >> - '"--option1 value1 [--option2 value2 [...]]"')) >> - return parser >> - >> - def __init__(self, out_file, options): >> - super(ResultsFormatter, self).__init__() >> - self.out_file = out_file >> - self.options = options >> - self.using_terminal = False >> - if not self.out_file: >> - raise Exception("ResultsFormatter created with no file >> object") >> - self.start_time_by_test = {} >> - self.terminate_called = False >> - >> - # Store counts of test_result events by status. >> - self.result_status_counts = { >> - EventBuilder.STATUS_SUCCESS: 0, >> - EventBuilder.STATUS_EXPECTED_FAILURE: 0, >> - EventBuilder.STATUS_SKIP: 0, >> - EventBuilder.STATUS_UNEXPECTED_SUCCESS: 0, >> - EventBuilder.STATUS_FAILURE: 0, >> - EventBuilder.STATUS_ERROR: 0 >> - } >> - >> - # Lock that we use while mutating inner state, like the >> - # total test count and the elements. We minimize how >> - # long we hold the lock just to keep inner state safe, not >> - # entirely consistent from the outside. >> - self.lock = threading.Lock() >> - >> - def handle_event(self, test_event): >> - """Handles the test event for collection into the formatter >> output. >> - >> - Derived classes may override this but should call down to this >> - implementation first. >> - >> - @param test_event the test event as formatted by one of the >> - event_for_* calls. >> - """ >> - # Keep track of whether terminate was received. We do this so >> - # that a process can call the 'terminate' event on its own, to >> - # close down a formatter at the appropriate time. Then the >> - # atexit() cleanup can call the "terminate if it hasn't been >> - # called yet". >> - if test_event is not None: >> - event_type = test_event.get("event", "") >> - if event_type == "terminate": >> - self.terminate_called = True >> - elif event_type == "test_result": >> - # Keep track of event counts per test result status type >> - status = test_event["status"] >> - self.result_status_counts[status] += 1 >> - >> - def track_start_time(self, test_class, test_name, start_time): >> - """tracks the start time of a test so elapsed time can be >> computed. >> - >> - this alleviates the need for test results to be processed >> serially >> - by test. it will save the start time for the test so that >> - elapsed_time_for_test() can compute the elapsed time properly. >> - """ >> - if test_class is None or test_name is None: >> - return >> - >> - test_key = "{}.{}".format(test_class, test_name) >> - with self.lock: >> - self.start_time_by_test[test_key] = start_time >> - >> - def elapsed_time_for_test(self, test_class, test_name, end_time): >> - """returns the elapsed time for a test. >> - >> - this function can only be called once per test and requires that >> - the track_start_time() method be called sometime prior to calling >> - this method. >> - """ >> - if test_class is None or test_name is None: >> - return -2.0 >> - >> - test_key = "{}.{}".format(test_class, test_name) >> - with self.lock: >> - if test_key not in self.start_time_by_test: >> - return -1.0 >> - else: >> - start_time = self.start_time_by_test[test_key] >> - del self.start_time_by_test[test_key] >> - return end_time - start_time >> - >> - def is_using_terminal(self): >> - """returns true if this results formatter is using the terminal >> and >> - output should be avoided.""" >> - return self.using_terminal >> - >> - def send_terminate_as_needed(self): >> - """sends the terminate event if it hasn't been received yet.""" >> - if not self.terminate_called: >> - terminate_event = EventBuilder.bare_event("terminate") >> - self.handle_event(terminate_event) >> - >> - # Derived classes may require self access >> - # pylint: disable=no-self-use >> - def replaces_summary(self): >> - """Returns whether the results formatter includes a summary >> - suitable to replace the old lldb test run results. >> - >> - @return True if the lldb test runner can skip its summary >> - generation when using this results formatter; False otherwise. >> - """ >> - return False >> - >> - def counts_by_test_result_status(self, status): >> - """Returns number of test method results for the given status. >> - >> - @status_result a test result status (e.g. success, fail, skip) >> - as defined by the EventBuilder.STATUS_* class members. >> - >> - @return an integer returning the number of test methods matching >> - the given test result status. >> - """ >> - return self.result_status_counts[status] >> - >> - >> -class XunitFormatter(ResultsFormatter): >> - """Provides xUnit-style formatted output. >> - """ >> - >> - # Result mapping arguments >> - RM_IGNORE = 'ignore' >> - RM_SUCCESS = 'success' >> - RM_FAILURE = 'failure' >> - RM_PASSTHRU = 'passthru' >> - >> - @staticmethod >> - def _build_illegal_xml_regex(): >> - """Contructs a regex to match all illegal xml characters. >> - >> - Expects to be used against a unicode string.""" >> - # Construct the range pairs of invalid unicode chareacters. >> - illegal_chars_u = [ >> - (0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x1F), (0x7F, 0x84), >> - (0x86, 0x9F), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF)] >> - >> - # For wide builds, we have more. >> - if sys.maxunicode >= 0x10000: >> - illegal_chars_u.extend( >> - [(0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, >> 0x3FFFF), >> - (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, >> 0x6FFFF), >> - (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, >> 0x9FFFF), >> - (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, >> 0xCFFFF), >> - (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, >> 0xFFFFF), >> - (0x10FFFE, 0x10FFFF)]) >> - >> - # Build up an array of range expressions. >> - illegal_ranges = [ >> - "%s-%s" % (six.unichr(low), six.unichr(high)) >> - for (low, high) in illegal_chars_u] >> - >> - # Compile the regex >> - return re.compile(six.u('[%s]') % six.u('').join(illegal_ranges)) >> - >> - @staticmethod >> - def _quote_attribute(text): >> - """Returns the given text in a manner safe for usage in an XML >> attribute. >> - >> - @param text the text that should appear within an XML attribute. >> - @return the attribute-escaped version of the input text. >> - """ >> - return xml.sax.saxutils.quoteattr(text) >> - >> - def _replace_invalid_xml(self, str_or_unicode): >> - """Replaces invalid XML characters with a '?'. >> - >> - @param str_or_unicode a string to replace invalid XML >> - characters within. Can be unicode or not. If not unicode, >> - assumes it is a byte string in utf-8 encoding. >> - >> - @returns a utf-8-encoded byte string with invalid >> - XML replaced with '?'. >> - """ >> - # Get the content into unicode >> - if isinstance(str_or_unicode, str): >> - unicode_content = str_or_unicode.decode('utf-8') >> - else: >> - unicode_content = str_or_unicode >> - return self.invalid_xml_re.sub( >> - six.u('?'), unicode_content).encode('utf-8') >> - >> - @classmethod >> - def arg_parser(cls): >> - """@return arg parser used to parse formatter-specific >> options.""" >> - parser = super(XunitFormatter, cls).arg_parser() >> - >> - # These are valid choices for results mapping. >> - results_mapping_choices = [ >> - XunitFormatter.RM_IGNORE, >> - XunitFormatter.RM_SUCCESS, >> - XunitFormatter.RM_FAILURE, >> - XunitFormatter.RM_PASSTHRU] >> - parser.add_argument( >> - "--assert-on-unknown-events", >> - action="store_true", >> - help=('cause unknown test events to generate ' >> - 'a python assert. Default is to ignore.')) >> - parser.add_argument( >> - "--ignore-skip-name", >> - "-n", >> - metavar='PATTERN', >> - action="append", >> - dest='ignore_skip_name_patterns', >> - help=('a python regex pattern, where ' >> - 'any skipped test with a test method name where regex ' >> - 'matches (via search) will be ignored for xUnit test ' >> - 'result purposes. Can be specified multiple times.')) >> - parser.add_argument( >> - "--ignore-skip-reason", >> - "-r", >> - metavar='PATTERN', >> - action="append", >> - dest='ignore_skip_reason_patterns', >> - help=('a python regex pattern, where ' >> - 'any skipped test with a skip reason where the regex ' >> - 'matches (via search) will be ignored for xUnit test ' >> - 'result purposes. Can be specified multiple times.')) >> - parser.add_argument( >> - "--xpass", action="store", choices=results_mapping_choices, >> - default=XunitFormatter.RM_FAILURE, >> - help=('specify mapping from unexpected success to >> jUnit/xUnit ' >> - 'result type')) >> - parser.add_argument( >> - "--xfail", action="store", choices=results_mapping_choices, >> - default=XunitFormatter.RM_IGNORE, >> - help=('specify mapping from expected failure to jUnit/xUnit ' >> - 'result type')) >> - return parser >> - >> - @staticmethod >> - def _build_regex_list_from_patterns(patterns): >> - """Builds a list of compiled regexes from option value. >> - >> - @param option string containing a comma-separated list of regex >> - patterns. Zero-length or None will produce an empty regex list. >> - >> - @return list of compiled regular expressions, empty if no >> - patterns provided. >> - """ >> - regex_list = [] >> - if patterns is not None: >> - for pattern in patterns: >> - regex_list.append(re.compile(pattern)) >> - return regex_list >> - >> - def __init__(self, out_file, options): >> - """Initializes the XunitFormatter instance. >> - @param out_file file-like object where formatted output is >> written. >> - @param options_dict specifies a dictionary of options for the >> - formatter. >> - """ >> - # Initialize the parent >> - super(XunitFormatter, self).__init__(out_file, options) >> - self.text_encoding = "UTF-8" >> - self.invalid_xml_re = XunitFormatter._build_illegal_xml_regex() >> - self.total_test_count = 0 >> - self.ignore_skip_name_regexes = ( >> - XunitFormatter._build_regex_list_from_patterns( >> - options.ignore_skip_name_patterns)) >> - self.ignore_skip_reason_regexes = ( >> - XunitFormatter._build_regex_list_from_patterns( >> - options.ignore_skip_reason_patterns)) >> - >> - self.elements = { >> - "successes": [], >> - "errors": [], >> - "failures": [], >> - "skips": [], >> - "unexpected_successes": [], >> - "expected_failures": [], >> - "all": [] >> - } >> - >> - self.status_handlers = { >> - EventBuilder.STATUS_SUCCESS: self._handle_success, >> - EventBuilder.STATUS_FAILURE: self._handle_failure, >> - EventBuilder.STATUS_ERROR: self._handle_error, >> - EventBuilder.STATUS_SKIP: self._handle_skip, >> - EventBuilder.STATUS_EXPECTED_FAILURE: >> - self._handle_expected_failure, >> - EventBuilder.STATUS_UNEXPECTED_SUCCESS: >> - self._handle_unexpected_success >> - } >> - >> - def handle_event(self, test_event): >> - super(XunitFormatter, self).handle_event(test_event) >> - >> - event_type = test_event["event"] >> - if event_type is None: >> - return >> - >> - if event_type == "terminate": >> - self._finish_output() >> - elif event_type == "test_start": >> - self.track_start_time( >> - test_event["test_class"], >> - test_event["test_name"], >> - test_event["event_time"]) >> - elif event_type == "test_result": >> - self._process_test_result(test_event) >> - else: >> - # This is an unknown event. >> - if self.options.assert_on_unknown_events: >> - raise Exception("unknown event type {} from {}\n".format( >> - event_type, test_event)) >> - >> - def _handle_success(self, test_event): >> - """Handles a test success. >> - @param test_event the test event to handle. >> - """ >> - result = self._common_add_testcase_entry(test_event) >> - with self.lock: >> - self.elements["successes"].append(result) >> - >> - def _handle_failure(self, test_event): >> - """Handles a test failure. >> - @param test_event the test event to handle. >> - """ >> - message = self._replace_invalid_xml(test_event["issue_message"]) >> - backtrace = self._replace_invalid_xml( >> - "".join(test_event.get("issue_backtrace", []))) >> - >> - result = self._common_add_testcase_entry( >> - test_event, >> - inner_content=( >> - '<failure type={} >> message={}><![CDATA[{}]]></failure>'.format( >> - >> XunitFormatter._quote_attribute(test_event["issue_class"]), >> - XunitFormatter._quote_attribute(message), >> - backtrace) >> - )) >> - with self.lock: >> - self.elements["failures"].append(result) >> - >> - def _handle_error(self, test_event): >> - """Handles a test error. >> - @param test_event the test event to handle. >> - """ >> - message = self._replace_invalid_xml(test_event["issue_message"]) >> - backtrace = self._replace_invalid_xml( >> - "".join(test_event.get("issue_backtrace", []))) >> - >> - result = self._common_add_testcase_entry( >> - test_event, >> - inner_content=( >> - '<error type={} >> message={}><![CDATA[{}]]></error>'.format( >> - >> XunitFormatter._quote_attribute(test_event["issue_class"]), >> - XunitFormatter._quote_attribute(message), >> - backtrace) >> - )) >> - with self.lock: >> - self.elements["errors"].append(result) >> - >> - @staticmethod >> - def _ignore_based_on_regex_list(test_event, test_key, regex_list): >> - """Returns whether to ignore a test event based on patterns. >> - >> - @param test_event the test event dictionary to check. >> - @param test_key the key within the dictionary to check. >> - @param regex_list a list of zero or more regexes. May contain >> - zero or more compiled regexes. >> - >> - @return True if any o the regex list match based on the >> - re.search() method; false otherwise. >> - """ >> - for regex in regex_list: >> - match = regex.search(test_event.get(test_key, '')) >> - if match: >> - return True >> - return False >> - >> - def _handle_skip(self, test_event): >> - """Handles a skipped test. >> - @param test_event the test event to handle. >> - """ >> - >> - # Are we ignoring this test based on test name? >> - if XunitFormatter._ignore_based_on_regex_list( >> - test_event, 'test_name', self.ignore_skip_name_regexes): >> - return >> - >> - # Are we ignoring this test based on skip reason? >> - if XunitFormatter._ignore_based_on_regex_list( >> - test_event, 'skip_reason', >> self.ignore_skip_reason_regexes): >> - return >> - >> - # We're not ignoring this test. Process the skip. >> - reason = self._replace_invalid_xml(test_event.get("skip_reason", >> "")) >> - result = self._common_add_testcase_entry( >> - test_event, >> - inner_content='<skipped message={} />'.format( >> - XunitFormatter._quote_attribute(reason))) >> - with self.lock: >> - self.elements["skips"].append(result) >> - >> - def _handle_expected_failure(self, test_event): >> - """Handles a test that failed as expected. >> - @param test_event the test event to handle. >> - """ >> - if self.options.xfail == XunitFormatter.RM_PASSTHRU: >> - # This is not a natively-supported junit/xunit >> - # testcase mode, so it might fail a validating >> - # test results viewer. >> - if "bugnumber" in test_event: >> - bug_id_attribute = 'bug-id={} '.format( >> - >> XunitFormatter._quote_attribute(test_event["bugnumber"])) >> - else: >> - bug_id_attribute = '' >> - >> - result = self._common_add_testcase_entry( >> - test_event, >> - inner_content=( >> - '<expected-failure {}type={} message={} />'.format( >> - bug_id_attribute, >> - XunitFormatter._quote_attribute( >> - test_event["issue_class"]), >> - XunitFormatter._quote_attribute( >> - test_event["issue_message"])) >> - )) >> - with self.lock: >> - self.elements["expected_failures"].append(result) >> - elif self.options.xfail == XunitFormatter.RM_SUCCESS: >> - result = self._common_add_testcase_entry(test_event) >> - with self.lock: >> - self.elements["successes"].append(result) >> - elif self.options.xfail == XunitFormatter.RM_FAILURE: >> - result = self._common_add_testcase_entry( >> - test_event, >> - inner_content='<failure type={} message={} />'.format( >> - >> XunitFormatter._quote_attribute(test_event["issue_class"]), >> - XunitFormatter._quote_attribute( >> - test_event["issue_message"]))) >> - with self.lock: >> - self.elements["failures"].append(result) >> - elif self.options.xfail == XunitFormatter.RM_IGNORE: >> - pass >> - else: >> - raise Exception( >> - "unknown xfail option: {}".format(self.options.xfail)) >> - >> - def _handle_unexpected_success(self, test_event): >> - """Handles a test that passed but was expected to fail. >> - @param test_event the test event to handle. >> - """ >> - if self.options.xpass == XunitFormatter.RM_PASSTHRU: >> - # This is not a natively-supported junit/xunit >> - # testcase mode, so it might fail a validating >> - # test results viewer. >> - result = self._common_add_testcase_entry( >> - test_event, >> - inner_content=("<unexpected-success />")) >> - with self.lock: >> - self.elements["unexpected_successes"].append(result) >> - elif self.options.xpass == XunitFormatter.RM_SUCCESS: >> - # Treat the xpass as a success. >> - result = self._common_add_testcase_entry(test_event) >> - with self.lock: >> - self.elements["successes"].append(result) >> - elif self.options.xpass == XunitFormatter.RM_FAILURE: >> - # Treat the xpass as a failure. >> - if "bugnumber" in test_event: >> - message = "unexpected success (bug_id:{})".format( >> - test_event["bugnumber"]) >> - else: >> - message = "unexpected success (bug_id:none)" >> - result = self._common_add_testcase_entry( >> - test_event, >> - inner_content='<failure type={} message={} />'.format( >> - >> XunitFormatter._quote_attribute("unexpected_success"), >> - XunitFormatter._quote_attribute(message))) >> - with self.lock: >> - self.elements["failures"].append(result) >> - elif self.options.xpass == XunitFormatter.RM_IGNORE: >> - # Ignore the xpass result as far as xUnit reporting goes. >> - pass >> - else: >> - raise Exception("unknown xpass option: {}".format( >> - self.options.xpass)) >> - >> - def _process_test_result(self, test_event): >> - """Processes the test_event known to be a test result. >> - >> - This categorizes the event appropriately and stores the data >> needed >> - to generate the final xUnit report. This method skips events >> that >> - cannot be represented in xUnit output. >> - """ >> - if "status" not in test_event: >> - raise Exception("test event dictionary missing 'status' key") >> - >> - status = test_event["status"] >> - if status not in self.status_handlers: >> - raise Exception("test event status '{}' unsupported".format( >> - status)) >> - >> - # Call the status handler for the test result. >> - self.status_handlers[status](test_event) >> - >> - def _common_add_testcase_entry(self, test_event, inner_content=None): >> - """Registers a testcase result, and returns the text created. >> - >> - The caller is expected to manage failure/skip/success counts >> - in some kind of appropriate way. This call simply constructs >> - the XML and appends the returned result to the self.all_results >> - list. >> - >> - @param test_event the test event dictionary. >> - >> - @param inner_content if specified, gets included in the >> <testcase> >> - inner section, at the point before stdout and stderr would be >> - included. This is where a <failure/>, <skipped/>, <error/>, etc. >> - could go. >> - >> - @return the text of the xml testcase element. >> - """ >> - >> - # Get elapsed time. >> - test_class = test_event["test_class"] >> - test_name = test_event["test_name"] >> - event_time = test_event["event_time"] >> - time_taken = self.elapsed_time_for_test( >> - test_class, test_name, event_time) >> - >> - # Plumb in stdout/stderr once we shift over to only test results. >> - test_stdout = '' >> - test_stderr = '' >> - >> - # Formulate the output xml. >> - if not inner_content: >> - inner_content = "" >> - result = ( >> - '<testcase classname="{}" name="{}" time="{:.3f}">' >> - '{}{}{}</testcase>'.format( >> - test_class, >> - test_name, >> - time_taken, >> - inner_content, >> - test_stdout, >> - test_stderr)) >> - >> - # Save the result, update total test count. >> - with self.lock: >> - self.total_test_count += 1 >> - self.elements["all"].append(result) >> - >> - return result >> - >> - def _finish_output_no_lock(self): >> - """Flushes out the report of test executions to form valid xml >> output. >> - >> - xUnit output is in XML. The reporting system cannot complete the >> - formatting of the output without knowing when there is no more >> input. >> - This call addresses notifcation of the completed test run and >> thus is >> - when we can finish off the report output. >> - """ >> - >> - # Figure out the counts line for the testsuite. If we have >> - # been counting either unexpected successes or expected >> - # failures, we'll output those in the counts, at the risk of >> - # being invalidated by a validating test results viewer. >> - # These aren't counted by default so they won't show up unless >> - # the user specified a formatter option to include them. >> - xfail_count = len(self.elements["expected_failures"]) >> - xpass_count = len(self.elements["unexpected_successes"]) >> - if xfail_count > 0 or xpass_count > 0: >> - extra_testsuite_attributes = ( >> - ' expected-failures="{}"' >> - ' unexpected-successes="{}"'.format(xfail_count, >> xpass_count)) >> - else: >> - extra_testsuite_attributes = "" >> - >> - # Output the header. >> - self.out_file.write( >> - '<?xml version="1.0" encoding="{}"?>\n' >> - '<testsuites>' >> - '<testsuite name="{}" tests="{}" errors="{}" failures="{}" ' >> - 'skip="{}"{}>\n'.format( >> - self.text_encoding, >> - "LLDB test suite", >> - self.total_test_count, >> - len(self.elements["errors"]), >> - len(self.elements["failures"]), >> - len(self.elements["skips"]), >> - extra_testsuite_attributes)) >> - >> - # Output each of the test result entries. >> - for result in self.elements["all"]: >> - self.out_file.write(result + '\n') >> - >> - # Close off the test suite. >> - self.out_file.write('</testsuite></testsuites>\n') >> - >> - def _finish_output(self): >> - """Finish writing output as all incoming events have arrived.""" >> - with self.lock: >> - self._finish_output_no_lock() >> - >> - >> -class RawPickledFormatter(ResultsFormatter): >> - """Formats events as a pickled stream. >> - >> - The parallel test runner has inferiors pickle their results and send >> them >> - over a socket back to the parallel test. The parallel test runner >> then >> - aggregates them into the final results formatter (e.g. xUnit). >> - """ >> - >> - @classmethod >> - def arg_parser(cls): >> - """@return arg parser used to parse formatter-specific >> options.""" >> - parser = super(RawPickledFormatter, cls).arg_parser() >> - return parser >> - >> - def __init__(self, out_file, options): >> - super(RawPickledFormatter, self).__init__(out_file, options) >> - self.pid = os.getpid() >> - >> - def handle_event(self, test_event): >> - super(RawPickledFormatter, self).handle_event(test_event) >> - >> - # Convert initialize/terminate events into job_begin/job_end >> events. >> - event_type = test_event["event"] >> - if event_type is None: >> - return >> - >> - if event_type == "initialize": >> - test_event["event"] = "job_begin" >> - elif event_type == "terminate": >> - test_event["event"] = "job_end" >> - >> - # Tack on the pid. >> - test_event["pid"] = self.pid >> - >> - # Send it as >> {serialized_length_of_serialized_bytes}{serialized_bytes} >> - import struct >> - msg = cPickle.dumps(test_event) >> - packet = struct.pack("!I%ds" % len(msg), len(msg), msg) >> - self.out_file.send(packet) >> - >> - >> -class DumpFormatter(ResultsFormatter): >> - """Formats events to the file as their raw python dictionary >> format.""" >> - >> - def handle_event(self, test_event): >> - super(DumpFormatter, self).handle_event(test_event) >> - self.out_file.write("\n" + pprint.pformat(test_event) + "\n") >> >> >> _______________________________________________ >> lldb-commits mailing list >> lldb-commits@lists.llvm.org >> http://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits >> > > > > -- > -Todd >
_______________________________________________ lldb-commits mailing list lldb-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
