Added expectedFlakeyClang/Gcc, switched a bunch of tests to use new decorator.
http://reviews.llvm.org/D10721
Files:
test/expression_command/call-function/TestCallStopAndContinue.py
test/expression_command/call-restarts/TestCallThatRestarts.py
test/expression_command/timeout/TestCallWithTimeout.py
test/functionalities/attach_resume/TestAttachResume.py
test/functionalities/avoids-fd-leak/TestFdLeak.py
test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py
test/functionalities/stop-hook/TestStopHookMechanism.py
test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py
test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py
test/lang/c/register_variables/TestRegisterVariables.py
test/lang/cpp/this/TestCPPThis.py
test/lldbtest.py
test/python_api/signals/TestSignalsAPI.py
test/types/TestFloatTypes.py
EMAIL PREFERENCES
http://reviews.llvm.org/settings/panel/emailpreferences/
Index: test/expression_command/call-function/TestCallStopAndContinue.py
===================================================================
--- test/expression_command/call-function/TestCallStopAndContinue.py
+++ test/expression_command/call-function/TestCallStopAndContinue.py
@@ -22,14 +22,14 @@
@skipUnlessDarwin
@dsym_test
- @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX
+ @expectedFlakeyDarwin("llvm.org/pr20274")
def test_with_dsym(self):
"""Test gathering result from interrupted function call."""
self.buildDsym()
self.call_function()
@dwarf_test
- @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX
+ @expectedFlakeyDarwin("llvm.org/pr20274")
def test_with_dwarf(self):
"""Test gathering result from interrupted function call."""
self.buildDwarf()
Index: test/expression_command/call-restarts/TestCallThatRestarts.py
===================================================================
--- test/expression_command/call-restarts/TestCallThatRestarts.py
+++ test/expression_command/call-restarts/TestCallThatRestarts.py
@@ -29,7 +29,7 @@
@dwarf_test
@skipIfFreeBSD # llvm.org/pr19246: intermittent failure
- @expectedFailureLinux("llvm.org/pr19246") # intermittent failure
+ @expectedFailureLinux("llvm.org/pr19246")
@skipIfDarwin # llvm.org/pr19246: intermittent failure
@skipIfWindows # Test relies on signals, unsupported on Windows
def test_with_dwarf(self):
Index: test/expression_command/timeout/TestCallWithTimeout.py
===================================================================
--- test/expression_command/timeout/TestCallWithTimeout.py
+++ test/expression_command/timeout/TestCallWithTimeout.py
@@ -28,7 +28,7 @@
self.call_function()
@expectedFailureFreeBSD("llvm.org/pr19605") # fails on buildbot
- @expectedFailureLinux("llvm.org/pr20275") # fails intermittently on Linux
+ @expectedFlakeyLinux("llvm.org/pr20275")
@dwarf_test
def test_with_dwarf(self):
"""Test calling std::String member function."""
Index: test/functionalities/attach_resume/TestAttachResume.py
===================================================================
--- test/functionalities/attach_resume/TestAttachResume.py
+++ test/functionalities/attach_resume/TestAttachResume.py
@@ -22,7 +22,7 @@
self.buildDwarf()
self.process_attach_continue_interrupt_detach()
- @expectedFailureLinux('llvm.org/pr19478') # intermittent ~2/14 runs
+ @expectedFlakeyLinux('llvm.org/pr19478') # intermittent ~2/14 runs
@skipIfRemote
def process_attach_continue_interrupt_detach(self):
"""Test attach/continue/interrupt/detach"""
Index: test/functionalities/avoids-fd-leak/TestFdLeak.py
===================================================================
--- test/functionalities/avoids-fd-leak/TestFdLeak.py
+++ test/functionalities/avoids-fd-leak/TestFdLeak.py
@@ -41,8 +41,8 @@
self.assertTrue(process.GetExitStatus() == 0,
"Process returned non-zero status. Were incorrect file descriptors passed?")
- @expectedFailure(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd
- @expectedFailureLinux # xfail flakey test to get buildbot green
+ @expectedFlakey(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd
+ @expectedFlakeyLinux
@skipIfWindows # The check for descriptor leakage needs to be implemented differently here.
@skipIfTargetAndroid() # Android have some other file descriptors open by the shell
def test_fd_leak_multitarget (self):
Index: test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py
===================================================================
--- test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py
+++ test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py
@@ -24,7 +24,7 @@
@dwarf_test
@expectedFailureFreeBSD('llvm.org/pr17946')
- @expectedFailureLinux # failed 1/365 test runs, line 61, thread.IsValid()
+ @expectedFlakeyLinux # failed 1/365 test runs, line 61, thread.IsValid()
def test_with_dwarf_and_run_command(self):
"""Test that expr will time out and allow other threads to run if it blocks."""
self.buildDwarf()
Index: test/functionalities/stop-hook/TestStopHookMechanism.py
===================================================================
--- test/functionalities/stop-hook/TestStopHookMechanism.py
+++ test/functionalities/stop-hook/TestStopHookMechanism.py
@@ -19,7 +19,7 @@
self.stop_hook_firing()
@skipIfFreeBSD # llvm.org/pr15037
- @expectedFailureLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux
+ @expectedFlakeyLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux
@expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
@dwarf_test
def test_with_dwarf(self):
Index: test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py
===================================================================
--- test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py
+++ test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py
@@ -21,7 +21,7 @@
@dwarf_test
@expectedFailureFreeBSD("llvm.org/pr15037")
- @expectedFailureLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux
+ @expectedFlakeyLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux
@expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
@skipIfFreeBSD # llvm.org/pr22784
def test_stop_hook_multiple_threads_with_dwarf(self):
Index: test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
===================================================================
--- test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
+++ test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
@@ -22,8 +22,8 @@
@skipIfFreeBSD # Hangs. May be the same as Linux issue llvm.org/pr16229 but
# not yet investigated. Revisit once required functionality
# is implemented for FreeBSD.
- @expectedFailureLinux("llvm.org/pr16229") # Occasionally hangs on the build bot
- @skipIfLinux # Occasionally hangs on the build bot
+ @expectedFlakeyLinux("llvm.org/pr16229")
+ @skipIfLinux # Occasionally hangs on the build bot, expectedFailureLinux
@dwarf_test
def test_create_after_attach_with_dwarf_and_popen(self):
"""Test thread creation after process attach."""
@@ -34,7 +34,7 @@
# for FreeBSD.
@dwarf_test
@skipIfRemote
- @expectedFailureLinux # this test fails 1/100 dosep runs
+ @expectedFlakeyLinux # this test fails 1/100 dosep runs
def test_create_after_attach_with_dwarf_and_fork(self):
"""Test thread creation after process attach."""
self.buildDwarf(dictionary=self.getBuildFlags(use_cpp11=False))
Index: test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py
===================================================================
--- test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py
+++ test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py
@@ -23,7 +23,7 @@
@expectedFailureFreeBSD('llvm.org/pr18522') # hits break in another thread in testrun
@python_api_test
@dwarf_test
- @expectedFailureLinux # this test fails 6/100 dosep runs
+ @expectedFlakeyLinux # this test fails 6/100 dosep runs
def test_with_dwarf_python(self):
"""Test that we obey thread conditioned breakpoints."""
self.buildDwarf()
Index: test/lang/c/register_variables/TestRegisterVariables.py
===================================================================
--- test/lang/c/register_variables/TestRegisterVariables.py
+++ test/lang/c/register_variables/TestRegisterVariables.py
@@ -19,7 +19,7 @@
@dwarf_test
@expectedFailureClang(None, ['<', '3.5'])
- @expectedFailureGcc #xfail to get buildbot green, test failed with gcc4.8.2
+ @expectedFlakeyGcc # failed with gcc4.8.2
def test_with_dwarf_and_run_command(self):
"""Test expressions on register values."""
self.buildDwarf()
Index: test/lang/cpp/this/TestCPPThis.py
===================================================================
--- test/lang/cpp/this/TestCPPThis.py
+++ test/lang/cpp/this/TestCPPThis.py
@@ -22,7 +22,7 @@
@expectedFailureGcc # llvm.org/pr15439 The 'this' pointer isn't available during expression evaluation when stopped in an inlined member function.
@expectedFailureIcc # ICC doesn't emit correct DWARF inline debug info for inlined member functions
@dwarf_test
- @expectedFailureClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6'])#xfail to get buildbot green, test failed with totclang - clang3.7
+ @expectedFlakeyClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6']) # failed with totclang - clang3.7
def test_with_dwarf_and_run_command(self):
"""Test that the appropriate member variables are available when stopped in C++ static, inline, and const methods"""
self.buildDwarf()
Index: test/lldbtest.py
===================================================================
--- test/lldbtest.py
+++ test/lldbtest.py
@@ -712,6 +712,68 @@
return expectedFailure(fn, bugnumber)
+# if the test passes on the first try, we're done (success)
+# if the test fails once, then passes on the second try, raise an ExpectedFailure
+# if the test fails twice in a row, re-throw the exception from the second test run
+def expectedFlakey(expected_fn, bugnumber=None):
+ def expectedFailure_impl(func):
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ from unittest2 import case
+ self = args[0]
+ try:
+ func(*args, **kwargs)
+ except Exception:
+ if expected_fn(self):
+ # retry
+ try:
+ func(*args, **kwargs)
+ except Exception:
+ # oh snap! two failures in a row, record a failure/error
+ raise
+ # record the expected failure
+ raise case._ExpectedFailure(sys.exc_info(), bugnumber)
+ else:
+ raise
+ return wrapper
+ # if bugnumber is not-callable(incluing None), that means decorator function is called with optional arguments
+ # return decorator in this case, so it will be used to decorating original method
+ if callable(bugnumber):
+ return expectedFailure_impl(bugnumber)
+ else:
+ return expectedFailure_impl
+
+def expectedFlakeyOS(oslist, bugnumber=None, compilers=None):
+ def fn(self):
+ return (self.getPlatform() in oslist and
+ self.expectedCompiler(compilers))
+ return expectedFlakey(fn, bugnumber)
+
+def expectedFlakeyDarwin(bugnumber=None, compilers=None):
+ # For legacy reasons, we support both "darwin" and "macosx" as OS X triples.
+ return expectedFlakeyOS(getDarwinOSTriples(), bugnumber, compilers)
+
+def expectedFlakeyLinux(bugnumber=None, compilers=None):
+ return expectedFlakeyOS(['linux'], bugnumber, compilers)
+
+def expectedFlakeyFreeBSD(bugnumber=None, compilers=None):
+ return expectedFlakeyOS(['freebsd'], bugnumber, compilers)
+
+def expectedFlakeyCompiler(compiler, compiler_version=None, bugnumber=None):
+ if compiler_version is None:
+ compiler_version=['=', None]
+ def fn(self):
+ return compiler in self.getCompiler() and self.expectedCompilerVersion(compiler_version)
+ return expectedFlakey(fn, bugnumber)
+
+# @expectedFlakeyClang('bugnumber', ['<=', '3.4'])
+def expectedFlakeyClang(bugnumber=None, compiler_version=None):
+ return expectedFlakeyCompiler('clang', compiler_version, bugnumber)
+
+# @expectedFlakeyGcc('bugnumber', ['<=', '3.4'])
+def expectedFlakeyGcc(bugnumber=None, compiler_version=None):
+ return expectedFlakeyCompiler('gcc', compiler_version, bugnumber)
+
def skipIfRemote(func):
"""Decorate the item to skip tests if testing remotely."""
if isinstance(func, type) and issubclass(func, unittest2.TestCase):
Index: test/python_api/signals/TestSignalsAPI.py
===================================================================
--- test/python_api/signals/TestSignalsAPI.py
+++ test/python_api/signals/TestSignalsAPI.py
@@ -12,7 +12,7 @@
mydir = TestBase.compute_mydir(__file__)
@python_api_test
- @expectedFailureLinux # this test fails 1/100 dosep runs
+ @expectedFlakeyLinux # this test fails 1/100 dosep runs
def test_ignore_signal(self):
"""Test Python SBUnixSignals.Suppress/Stop/Notify() API."""
self.buildDefault()
Index: test/types/TestFloatTypes.py
===================================================================
--- test/types/TestFloatTypes.py
+++ test/types/TestFloatTypes.py
@@ -38,7 +38,7 @@
@skipUnlessDarwin
@dsym_test
- @expectedFailureDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110
+ @expectedFlakeyDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110
def test_double_type_with_dsym(self):
"""Test that double-type variables are displayed correctly."""
self.build_and_run('double.cpp', set(['double']))
_______________________________________________
lldb-commits mailing list
lldb-commits@cs.uiuc.edu
http://lists.cs.uiuc.edu/mailman/listinfo/lldb-commits
