https://github.com/qiyao updated https://github.com/llvm/llvm-project/pull/214768
>From 6606f7f236ed6c534b59c4f60ab930cefb12eca0 Mon Sep 17 00:00:00 2001 From: Yao Qi <[email protected]> Date: Thu, 6 Aug 2026 23:15:57 +0100 Subject: [PATCH 1/3] [lldb] Gate debugserver tests on advertised qSupported features Tests that need a recent `debugserver` skipped themselves with `@skipIfOutOfTreeDebugserver`, which asks whether the stub was built in the tree rather than whether it supports the feature under test. A system `debugserver` that already ships the feature was skipped anyway, and a stale in-tree build was not. Every year, we remove some `@skipIfOutOfTreeDebugserver` because the new feature shipped in system `debugserver`. Ask the stub instead. Tests get the supported features of stub, and only run the test if stub advertises the very feature. `TestConsecutiveWatchpoints` keeps the decorator, because it covers a `debugserver` bug fix that no capability describes and it also runs against `lldb-server`. A stub without the feature can never run the test, so the reason is an `UnsupportedReason` and the result is `UNSUPPORTED` rather than `SKIPPED`. Against a system `debugserver`, the gated test in `TestExpeditedStackMemory.py` used to report ``` SKIPPED: ... test_memory_reads_when_examining_frame0_locals (out-of-tree debugserver) Skip breakdown (unsupported=0, skipped=1) ``` and now reports ``` UNSUPPORTED: ... test_memory_reads_when_examining_frame0_locals (stub does not support ExpediteStack+) Skip breakdown (unsupported=1, skipped=0) ``` These are the first skips decided while the test is running, and a reason raised from a test body reaches the result as `str(exception)`, which drops the `UnsupportedReason` type. `Base.skipTest` therefore records the distinction on the test, and `is_unsupported()` accepts it from either source: the reason object for a `require*` decorator, the test for a runtime skip. --- lldb/packages/Python/lldbsuite/test/lldbtest.py | 10 ++++++++++ lldb/packages/Python/lldbsuite/test/lldbutil.py | 8 ++++++++ lldb/packages/Python/lldbsuite/test/skip_reason.py | 6 ++++-- lldb/packages/Python/lldbsuite/test/test_result.py | 4 ++-- .../test/tools/lldb-server/gdbremote_testcase.py | 1 + .../multi-breakpoint/TestMultiBreakpoint.py | 6 ++---- .../API/lang/objc/foundation/TestObjCMethodsNSError.py | 3 ++- .../TestDebugserverMultiMemRead.py | 4 +--- .../expedited-stack-memory/TestExpeditedStackMemory.py | 6 +++++- .../shared-cache-vm-range/TestSharedCacheVMRange.py | 8 ++++++-- lldb/test/API/test_utils/TestDecorators.py | 10 ++++++++++ lldb/tools/debugserver/source/RNBRemote.cpp | 2 ++ 12 files changed, 53 insertions(+), 15 deletions(-) diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py index a78c11c5752d5..9bc148436a4b5 100644 --- a/lldb/packages/Python/lldbsuite/test/lldbtest.py +++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py @@ -63,6 +63,7 @@ from . import test_categories from lldbsuite.support import encoded_file from lldbsuite.support import funcutils +from lldbsuite.test.skip_reason import UnsupportedReason from lldbsuite.test_event import build_exception # See also dotest.parseOptionsAndInitTestdirs(), where the environment variables @@ -1291,6 +1292,15 @@ def markExpectedFailure(self, err): # Once by the Python unittest framework, and a second time by us. print("expected failure", file=sbuf) + def skipTest(self, reason): + """Skip the test, reporting an `UnsupportedReason` as UNSUPPORTED. + + `unittest` records a raised `SkipTest` as `str(exception)`, so the reason + type never reaches the result; remember the distinction on the test.""" + if isinstance(reason, UnsupportedReason): + self._skipped_as_unsupported = True + super().skipTest(reason) + def markSkippedTest(self): """Callback invoked when a test is skipped.""" self.__skipped__ = True diff --git a/lldb/packages/Python/lldbsuite/test/lldbutil.py b/lldb/packages/Python/lldbsuite/test/lldbutil.py index 78f7155ee2507..2154790a52742 100644 --- a/lldb/packages/Python/lldbsuite/test/lldbutil.py +++ b/lldb/packages/Python/lldbsuite/test/lldbutil.py @@ -22,6 +22,7 @@ from . import lldbtest_config from . import configuration from lldbsuite.test.gdbclientutils import escape_binary +from lldbsuite.test.skip_reason import UnsupportedReason # How often failed simulator process launches are retried. SIMULATOR_RETRY = 3 @@ -2008,6 +2009,13 @@ def get_qsupported_capabilities(test): return reply.strip().split(";") +def require_qsupported_capability(test, capability): + """Mark the test UNSUPPORTED unless the stub advertises *capability* in its + qSupported reply. Requires a live process.""" + if capability not in get_qsupported_capabilities(test): + test.skipTest(UnsupportedReason(f"stub does not support {capability}")) + + def connect_to_new_remote_platform(testcase, platform_exe, extra_args=[]): hostname = socket.getaddrinfo("localhost", 0, proto=socket.IPPROTO_TCP)[0][4][0] port_file = testcase.getBuildArtifact("port") diff --git a/lldb/packages/Python/lldbsuite/test/skip_reason.py b/lldb/packages/Python/lldbsuite/test/skip_reason.py index b1aece572f4f3..a237633c81514 100644 --- a/lldb/packages/Python/lldbsuite/test/skip_reason.py +++ b/lldb/packages/Python/lldbsuite/test/skip_reason.py @@ -12,6 +12,8 @@ class UnsupportedReason(str): broken here". Reported as UNSUPPORTED rather than SKIPPED.""" -def is_unsupported(reason): +def is_unsupported(test, reason): """Return True if *reason* marks a test as unsupported rather than skipped.""" - return isinstance(reason, UnsupportedReason) + return isinstance(reason, UnsupportedReason) or getattr( + test, "_skipped_as_unsupported", False + ) diff --git a/lldb/packages/Python/lldbsuite/test/test_result.py b/lldb/packages/Python/lldbsuite/test/test_result.py index ddd4d2ab3aaef..81670ec4c9f1c 100644 --- a/lldb/packages/Python/lldbsuite/test/test_result.py +++ b/lldb/packages/Python/lldbsuite/test/test_result.py @@ -169,7 +169,7 @@ def hardMarkAsSkipped(self, test): def countUnsupported(self): """Number of skipped tests that can never run in this configuration.""" - return sum(1 for _, reason in self.skipped if is_unsupported(reason)) + return sum(1 for test, reason in self.skipped if is_unsupported(test, reason)) def countSkipped(self): """Number of skipped tests that ought to run here but don't work yet.""" @@ -294,7 +294,7 @@ def addSkip(self, test, reason): # A test turned off by a `require*` decorator can never run in this # configuration, so report it as UNSUPPORTED. Anything else is a test # that ought to run here but doesn't work yet: report it as SKIPPED. - status = "UNSUPPORTED" if is_unsupported(reason) else "SKIPPED" + status = "UNSUPPORTED" if is_unsupported(test, reason) else "SKIPPED" self.stream.write( "%s: LLDB (%s) :: %s (%s) \n" % (status, self._config_string(test), str(test), reason) diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py index ede6ea8490951..99f278c3186d1 100644 --- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py +++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py @@ -952,6 +952,7 @@ def add_qSupported_packets(self, client_features=[]): "MultiMemRead", "jMultiBreakpoint", "accelerator-plugins", + "ExpediteStack", ] def parse_qSupported_response(self, context): diff --git a/lldb/test/API/functionalities/multi-breakpoint/TestMultiBreakpoint.py b/lldb/test/API/functionalities/multi-breakpoint/TestMultiBreakpoint.py index a4d6351e05d65..d8ff9613ec368 100644 --- a/lldb/test/API/functionalities/multi-breakpoint/TestMultiBreakpoint.py +++ b/lldb/test/API/functionalities/multi-breakpoint/TestMultiBreakpoint.py @@ -13,7 +13,6 @@ @skipIfWindows # No server on Windows. -@skipIfOutOfTreeDebugserver # Runs on systems where we can always predict the software break size @skipIf(archs=no_match(["x86_64", "arm64", "aarch64"])) class TestMultiBreakpoint(TestBase): @@ -71,9 +70,8 @@ def test_multi_breakpoint(self): self, "break here", source_file ) - # Verify the server advertises jMultiBreakpoint support. - capabilities = lldbutil.get_qsupported_capabilities(self) - self.assertIn("jMultiBreakpoint+", capabilities) + # The stub must advertise jMultiBreakpoint support. + lldbutil.require_qsupported_capability(self, "jMultiBreakpoint+") addr_a = self.get_function_address("func_a") addr_b = self.get_function_address("func_b") diff --git a/lldb/test/API/lang/objc/foundation/TestObjCMethodsNSError.py b/lldb/test/API/lang/objc/foundation/TestObjCMethodsNSError.py index a9fbe54e074ff..866a498693be5 100644 --- a/lldb/test/API/lang/objc/foundation/TestObjCMethodsNSError.py +++ b/lldb/test/API/lang/objc/foundation/TestObjCMethodsNSError.py @@ -46,7 +46,6 @@ def test_NSError_p(self): ) self.runCmd("process continue") - @skipIfOutOfTreeDebugserver def test_runtime_types_efficient_memreads(self): # Test that we use an efficient reading of memory when reading # Objective-C method descriptions. @@ -59,6 +58,8 @@ def test_runtime_types_efficient_memreads(self): self, "// Break here for NSString tests", lldb.SBFileSpec("main.m", False) ) + lldbutil.require_qsupported_capability(self, "MultiMemRead+") + self.runCmd(f"proc plugin packet send StartTesting", check=False) self.expect('expression str = [NSString stringWithCString: "new"]') self.runCmd(f"proc plugin packet send EndTesting", check=False) diff --git a/lldb/test/API/macosx/debugserver-multimemread/TestDebugserverMultiMemRead.py b/lldb/test/API/macosx/debugserver-multimemread/TestDebugserverMultiMemRead.py index 09dfbfe63216e..c0f044fb7eb69 100644 --- a/lldb/test/API/macosx/debugserver-multimemread/TestDebugserverMultiMemRead.py +++ b/lldb/test/API/macosx/debugserver-multimemread/TestDebugserverMultiMemRead.py @@ -9,7 +9,6 @@ @requireDarwin -@skipIfOutOfTreeDebugserver class TestCase(TestBase): def check_invalid_packet(self, packet_str): reply = lldbutil.send_packet_get_reply(self, "packet_str") @@ -25,8 +24,7 @@ def test_packets(self): self, "break here", source_file ) - capabilities = lldbutil.get_qsupported_capabilities(self) - self.assertIn("MultiMemRead+", capabilities) + lldbutil.require_qsupported_capability(self, "MultiMemRead+") mem_address_var = thread.frames[0].FindVariable("memory") self.assertTrue(mem_address_var) diff --git a/lldb/test/API/macosx/expedited-stack-memory/TestExpeditedStackMemory.py b/lldb/test/API/macosx/expedited-stack-memory/TestExpeditedStackMemory.py index 6fb31be4f379e..6bb64b1e6c6ee 100644 --- a/lldb/test/API/macosx/expedited-stack-memory/TestExpeditedStackMemory.py +++ b/lldb/test/API/macosx/expedited-stack-memory/TestExpeditedStackMemory.py @@ -47,7 +47,6 @@ def test_memory_reads_during_backtrace_without_cache(self): stub, producing memory-read packets.""" self.check_packets_during_backtrace(disable_memory_cache=True) - @skipIfOutOfTreeDebugserver @requireDarwin def test_memory_reads_when_examining_frame0_locals(self): """Model an IDE stop: walk the whole stack (a backtrace / debug @@ -93,6 +92,11 @@ def per_frame(idx, frame): sent = self.walk_stack(per_frame, disable_memory_cache=False) + if not expect_stack_reads: + # Only a stub that expedites frame 0's stack can serve its locals + # from the cache. + lldbutil.require_qsupported_capability(self, "ExpediteStack+") + # The process is still stopped after the walk; consult its memory map to # classify the reads we just provoked. Frame 0 is func_e (where we # stopped), so both the stack pointer and the `heap` local live there. diff --git a/lldb/test/API/macosx/shared-cache-vm-range/TestSharedCacheVMRange.py b/lldb/test/API/macosx/shared-cache-vm-range/TestSharedCacheVMRange.py index f823ef6b24e1b..9ead8f65b5174 100644 --- a/lldb/test/API/macosx/shared-cache-vm-range/TestSharedCacheVMRange.py +++ b/lldb/test/API/macosx/shared-cache-vm-range/TestSharedCacheVMRange.py @@ -8,6 +8,7 @@ from lldbsuite.test.decorators import * from lldbsuite.test.lldbtest import * from lldbsuite.test import lldbutil +from lldbsuite.test.skip_reason import UnsupportedReason class SharedCacheVMRangeTestCase(TestBase): @@ -15,7 +16,6 @@ class SharedCacheVMRangeTestCase(TestBase): @skipIfRemote @requireDarwin - @skipIfOutOfTreeDebugserver # debugserver returns shared_cache_size def test_shared_cache_vm_range(self): """Test that the shared cache VM range contains a known libc function""" self.build() @@ -44,7 +44,11 @@ def test_shared_cache_vm_range(self): response = re.search("response: (.+)", res.GetOutput()).group(1) json_response = json.loads(response) self.assertTrue("shared_cache_base_address" in json_response) - self.assertTrue("shared_cache_size" in json_response) + # Older debugservers don't report the size, so the range is unknown. + if "shared_cache_size" not in json_response: + self.skipTest( + UnsupportedReason("debugserver does not report shared_cache_size") + ) start = json_response["shared_cache_base_address"] end = start + json_response["shared_cache_size"] diff --git a/lldb/test/API/test_utils/TestDecorators.py b/lldb/test/API/test_utils/TestDecorators.py index 4810e281c8e79..e87c5b41ac4c2 100644 --- a/lldb/test/API/test_utils/TestDecorators.py +++ b/lldb/test/API/test_utils/TestDecorators.py @@ -1,7 +1,9 @@ import re +import unittest from lldbsuite.test.lldbtest import TestBase from lldbsuite.test.decorators import * +from lldbsuite.test.skip_reason import UnsupportedReason, is_unsupported def expectedFailureDwarf(bugnumber=None): @@ -113,3 +115,11 @@ def test_xfail_condition_true(self): @expectedFailureIf(condition=False) def test_xfail_condition_false(self): pass + + def test_unsupported_reason_survives_skip_test(self): + """A runtime skipTest(UnsupportedReason(...)) must report UNSUPPORTED.""" + try: + self.skipTest(UnsupportedReason("probe")) + except unittest.SkipTest as skip: + self.assertTrue(is_unsupported(self, str(skip))) + self._skipped_as_unsupported = False # this test passes, don't mark it diff --git a/lldb/tools/debugserver/source/RNBRemote.cpp b/lldb/tools/debugserver/source/RNBRemote.cpp index bc4ba6dc10f37..523307fb526f3 100644 --- a/lldb/tools/debugserver/source/RNBRemote.cpp +++ b/lldb/tools/debugserver/source/RNBRemote.cpp @@ -3816,6 +3816,8 @@ rnb_err_t RNBRemote::HandlePacket_qSupported(const char *p) { reply << "MultiMemRead+;"; reply << "jMultiBreakpoint+;"; + // The stopped thread's frame 0 stack memory is expedited in jThreadsInfo. + reply << "ExpediteStack+;"; return SendPacket(reply.str().c_str()); } >From bc7d38a281928c9682650f33a53e1f68014fa5df Mon Sep 17 00:00:00 2001 From: Yao Qi <[email protected]> Date: Fri, 14 Aug 2026 11:33:00 +0100 Subject: [PATCH 2/3] [lldb] Keep TestSharedCacheVMRange gated on skipIfOutOfTreeDebugserver `shared_cache_size` is not advertised in `qSupported`; the only way to notice it's missing is to send `jGetSharedCacheInfo` and look, and there a missing key is indistinguishable from a `debugserver` regression that dropped it. Keep the hard assertion and the decorator, so the key going away fails the test instead of silently skipping it. --- .../shared-cache-vm-range/TestSharedCacheVMRange.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lldb/test/API/macosx/shared-cache-vm-range/TestSharedCacheVMRange.py b/lldb/test/API/macosx/shared-cache-vm-range/TestSharedCacheVMRange.py index 9ead8f65b5174..f823ef6b24e1b 100644 --- a/lldb/test/API/macosx/shared-cache-vm-range/TestSharedCacheVMRange.py +++ b/lldb/test/API/macosx/shared-cache-vm-range/TestSharedCacheVMRange.py @@ -8,7 +8,6 @@ from lldbsuite.test.decorators import * from lldbsuite.test.lldbtest import * from lldbsuite.test import lldbutil -from lldbsuite.test.skip_reason import UnsupportedReason class SharedCacheVMRangeTestCase(TestBase): @@ -16,6 +15,7 @@ class SharedCacheVMRangeTestCase(TestBase): @skipIfRemote @requireDarwin + @skipIfOutOfTreeDebugserver # debugserver returns shared_cache_size def test_shared_cache_vm_range(self): """Test that the shared cache VM range contains a known libc function""" self.build() @@ -44,11 +44,7 @@ def test_shared_cache_vm_range(self): response = re.search("response: (.+)", res.GetOutput()).group(1) json_response = json.loads(response) self.assertTrue("shared_cache_base_address" in json_response) - # Older debugservers don't report the size, so the range is unknown. - if "shared_cache_size" not in json_response: - self.skipTest( - UnsupportedReason("debugserver does not report shared_cache_size") - ) + self.assertTrue("shared_cache_size" in json_response) start = json_response["shared_cache_base_address"] end = start + json_response["shared_cache_size"] >From 429be2021e3a759c67252c46cedeb825db163d6d Mon Sep 17 00:00:00 2001 From: Yao Qi <[email protected]> Date: Sun, 16 Aug 2026 13:10:21 +0100 Subject: [PATCH 3/3] [lldb][test] Fail when our own stub lacks the required capability A stub built from this tree must advertise the capability under test, so a missing one is a regression in the advertisement, not a reason to skip. Only an out-of-tree stub can legitimately lack it. --- lldb/packages/Python/lldbsuite/test/lldbutil.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lldb/packages/Python/lldbsuite/test/lldbutil.py b/lldb/packages/Python/lldbsuite/test/lldbutil.py index 2154790a52742..a29bba6c8e05d 100644 --- a/lldb/packages/Python/lldbsuite/test/lldbutil.py +++ b/lldb/packages/Python/lldbsuite/test/lldbutil.py @@ -2010,10 +2010,14 @@ def get_qsupported_capabilities(test): def require_qsupported_capability(test, capability): - """Mark the test UNSUPPORTED unless the stub advertises *capability* in its - qSupported reply. Requires a live process.""" - if capability not in get_qsupported_capabilities(test): - test.skipTest(UnsupportedReason(f"stub does not support {capability}")) + """Require *capability* in the stub's qSupported reply. Requires a live + process. Our own stub must advertise it, so a miss is a failure; a stub we + did not build can lack the feature, and the test is UNSUPPORTED.""" + if capability in get_qsupported_capabilities(test): + return + if not lldbtest_config.out_of_tree_debugserver: + test.fail(f"stub built from this tree does not advertise {capability}") + test.skipTest(UnsupportedReason(f"stub does not support {capability}")) def connect_to_new_remote_platform(testcase, platform_exe, extra_args=[]): _______________________________________________ lldb-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
