[Lldb-commits] [PATCH] D79607: [lldb/test][Darwin] Ask dyld where the real python is
This revision was automatically updated to reflect the committed changes.
Closed by commit rG8cb86ead7741: [lldb/test][Darwin] Ask dyld where the real
python is (authored by vsk).
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D79607/new/
https://reviews.llvm.org/D79607
Files:
lldb/test/API/get_darwin_real_python.py
lldb/test/API/lldbtest.py
Index: lldb/test/API/lldbtest.py
===
--- lldb/test/API/lldbtest.py
+++ lldb/test/API/lldbtest.py
@@ -3,6 +3,7 @@
import tempfile
import subprocess
import sys
+import platform
import lit.Test
import lit.TestRunner
@@ -75,19 +76,21 @@
builddir = getBuildDir(cmd)
mkdir_p(builddir)
-# The macOS system integrity protection (SIP) doesn't allow injecting
-# libraries into system binaries, but this can be worked around by
-# copying the binary into a different location.
+# On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim
+# python binary as the ASan interceptors get loaded too late. Also,
+# when SIP is enabled, we can't inject libraries into system binaries
+# at all, so we need a copy of the "real" python to work with.
+#
+# Find the "real" python binary, copy it, and invoke it.
if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \
-(executable.startswith('/System/') or \
-executable.startswith('/usr/bin/')):
+platform.system() == 'Darwin':
copied_python = os.path.join(builddir, 'copied-system-python')
if not os.path.isfile(copied_python):
import shutil, subprocess
python = subprocess.check_output([
executable,
-'-c',
-'import sys; print(sys.executable)'
+os.path.join(os.path.dirname(os.path.realpath(__file__)),
+'get_darwin_real_python.py')
]).decode('utf-8').strip()
shutil.copy(python, copied_python)
cmd[0] = copied_python
Index: lldb/test/API/get_darwin_real_python.py
===
--- /dev/null
+++ lldb/test/API/get_darwin_real_python.py
@@ -0,0 +1,14 @@
+# On macOS, system python binaries like /usr/bin/python and $(xcrun -f python3)
+# are shims. They do some light validation work and then spawn the "real"
python
+# binary. Find the "real" python by asking dyld -- sys.executable reports the
+# wrong thing more often than not. This is also useful when we're running under
+# a Homebrew python3 binary, which also appears to be some kind of shim.
+def getDarwinRealPythonExecutable():
+import ctypes
+dyld = ctypes.cdll.LoadLibrary('/usr/lib/system/libdyld.dylib')
+namelen = ctypes.c_ulong(1024)
+name = ctypes.create_string_buffer(b'\000', namelen.value)
+dyld._NSGetExecutablePath(ctypes.byref(name), ctypes.byref(namelen))
+return name.value.decode('utf-8').strip()
+
+print(getDarwinRealPythonExecutable())
Index: lldb/test/API/lldbtest.py
===
--- lldb/test/API/lldbtest.py
+++ lldb/test/API/lldbtest.py
@@ -3,6 +3,7 @@
import tempfile
import subprocess
import sys
+import platform
import lit.Test
import lit.TestRunner
@@ -75,19 +76,21 @@
builddir = getBuildDir(cmd)
mkdir_p(builddir)
-# The macOS system integrity protection (SIP) doesn't allow injecting
-# libraries into system binaries, but this can be worked around by
-# copying the binary into a different location.
+# On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim
+# python binary as the ASan interceptors get loaded too late. Also,
+# when SIP is enabled, we can't inject libraries into system binaries
+# at all, so we need a copy of the "real" python to work with.
+#
+# Find the "real" python binary, copy it, and invoke it.
if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \
-(executable.startswith('/System/') or \
-executable.startswith('/usr/bin/')):
+platform.system() == 'Darwin':
copied_python = os.path.join(builddir, 'copied-system-python')
if not os.path.isfile(copied_python):
import shutil, subprocess
python = subprocess.check_output([
executable,
-'-c',
-'import sys; print(sys.executable)'
+os.path.join(os.path.dirname(os.path.realpath(__file__)),
+'get_darwin_real_python.py')
]).decode('utf-8').strip()
shutil.copy(python, copied_python)
cmd[0] = copied_python
Index:
[Lldb-commits] [PATCH] D79607: [lldb/test][Darwin] Ask dyld where the real python is
JDevlieghere accepted this revision. JDevlieghere added a comment. This revision is now accepted and ready to land. LGTM! Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D79607/new/ https://reviews.llvm.org/D79607 ___ lldb-commits mailing list lldb-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
[Lldb-commits] [PATCH] D79607: [lldb/test][Darwin] Ask dyld where the real python is
vsk updated this revision to Diff 262799.
vsk added a comment.
- Reinstated comment about SIP to explain why we copy python
- Added a platform check
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D79607/new/
https://reviews.llvm.org/D79607
Files:
lldb/test/API/get_darwin_real_python.py
lldb/test/API/lldbtest.py
Index: lldb/test/API/lldbtest.py
===
--- lldb/test/API/lldbtest.py
+++ lldb/test/API/lldbtest.py
@@ -3,6 +3,7 @@
import tempfile
import subprocess
import sys
+import platform
import lit.Test
import lit.TestRunner
@@ -75,19 +76,21 @@
builddir = getBuildDir(cmd)
mkdir_p(builddir)
-# The macOS system integrity protection (SIP) doesn't allow injecting
-# libraries into system binaries, but this can be worked around by
-# copying the binary into a different location.
+# On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim
+# python binary as the ASan interceptors get loaded too late. Also,
+# when SIP is enabled, we can't inject libraries into system binaries
+# at all, so we need a copy of the "real" python to work with.
+#
+# Find the "real" python binary, copy it, and invoke it.
if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \
-(executable.startswith('/System/') or \
-executable.startswith('/usr/bin/')):
+platform.system() == 'Darwin':
copied_python = os.path.join(builddir, 'copied-system-python')
if not os.path.isfile(copied_python):
import shutil, subprocess
python = subprocess.check_output([
executable,
-'-c',
-'import sys; print(sys.executable)'
+os.path.join(os.path.dirname(os.path.realpath(__file__)),
+'get_darwin_real_python.py')
]).decode('utf-8').strip()
shutil.copy(python, copied_python)
cmd[0] = copied_python
Index: lldb/test/API/get_darwin_real_python.py
===
--- /dev/null
+++ lldb/test/API/get_darwin_real_python.py
@@ -0,0 +1,14 @@
+# On macOS, system python binaries like /usr/bin/python and $(xcrun -f python3)
+# are shims. They do some light validation work and then spawn the "real"
python
+# binary. Find the "real" python by asking dyld -- sys.executable reports the
+# wrong thing more often than not. This is also useful when we're running under
+# a Homebrew python3 binary, which also appears to be some kind of shim.
+def getDarwinRealPythonExecutable():
+import ctypes
+dyld = ctypes.cdll.LoadLibrary('/usr/lib/system/libdyld.dylib')
+namelen = ctypes.c_ulong(1024)
+name = ctypes.create_string_buffer(b'\000', namelen.value)
+dyld._NSGetExecutablePath(ctypes.byref(name), ctypes.byref(namelen))
+return name.value.decode('utf-8').strip()
+
+print(getDarwinRealPythonExecutable())
Index: lldb/test/API/lldbtest.py
===
--- lldb/test/API/lldbtest.py
+++ lldb/test/API/lldbtest.py
@@ -3,6 +3,7 @@
import tempfile
import subprocess
import sys
+import platform
import lit.Test
import lit.TestRunner
@@ -75,19 +76,21 @@
builddir = getBuildDir(cmd)
mkdir_p(builddir)
-# The macOS system integrity protection (SIP) doesn't allow injecting
-# libraries into system binaries, but this can be worked around by
-# copying the binary into a different location.
+# On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim
+# python binary as the ASan interceptors get loaded too late. Also,
+# when SIP is enabled, we can't inject libraries into system binaries
+# at all, so we need a copy of the "real" python to work with.
+#
+# Find the "real" python binary, copy it, and invoke it.
if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \
-(executable.startswith('/System/') or \
-executable.startswith('/usr/bin/')):
+platform.system() == 'Darwin':
copied_python = os.path.join(builddir, 'copied-system-python')
if not os.path.isfile(copied_python):
import shutil, subprocess
python = subprocess.check_output([
executable,
-'-c',
-'import sys; print(sys.executable)'
+os.path.join(os.path.dirname(os.path.realpath(__file__)),
+'get_darwin_real_python.py')
]).decode('utf-8').strip()
shutil.copy(python, copied_python)
cmd[0] = copied_python
Index: lldb/test/API/get_darwin_real_python.py
[Lldb-commits] [PATCH] D79607: [lldb/test][Darwin] Ask dyld where the real python is
JDevlieghere added inline comments.
Comment at: lldb/test/API/lldbtest.py:78
-# The macOS system integrity protection (SIP) doesn't allow injecting
-# libraries into system binaries, but this can be worked around by
-# copying the binary into a different location.
-if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \
-(executable.startswith('/System/') or \
-executable.startswith('/usr/bin/')):
+# On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim
+# python binary as the ASan interceptors get loaded too late. Find the
SIP still matters here, which is why we need to copy the python binary and not
just invoke it with the result from get_darwin_real_python.py. Can you merge
the two comments?
I also think that we might set DYLD_INSERT_LIBRARIES unconditionally on other
hosts. Can we guard this whole thing with a check for macOS?
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D79607/new/
https://reviews.llvm.org/D79607
___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
[Lldb-commits] [PATCH] D79607: [lldb/test][Darwin] Ask dyld where the real python is
vsk updated this revision to Diff 262792.
vsk added a comment.
Query dyld while running within the right python process.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D79607/new/
https://reviews.llvm.org/D79607
Files:
lldb/test/API/get_darwin_real_python.py
lldb/test/API/lldbtest.py
Index: lldb/test/API/lldbtest.py
===
--- lldb/test/API/lldbtest.py
+++ lldb/test/API/lldbtest.py
@@ -75,19 +75,17 @@
builddir = getBuildDir(cmd)
mkdir_p(builddir)
-# The macOS system integrity protection (SIP) doesn't allow injecting
-# libraries into system binaries, but this can be worked around by
-# copying the binary into a different location.
-if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \
-(executable.startswith('/System/') or \
-executable.startswith('/usr/bin/')):
+# On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim
+# python binary as the ASan interceptors get loaded too late. Find the
+# "real" python binary, copy it, and invoke it.
+if 'DYLD_INSERT_LIBRARIES' in test.config.environment:
copied_python = os.path.join(builddir, 'copied-system-python')
if not os.path.isfile(copied_python):
import shutil, subprocess
python = subprocess.check_output([
executable,
-'-c',
-'import sys; print(sys.executable)'
+os.path.join(os.path.dirname(os.path.realpath(__file__)),
+'get_darwin_real_python.py')
]).decode('utf-8').strip()
shutil.copy(python, copied_python)
cmd[0] = copied_python
Index: lldb/test/API/get_darwin_real_python.py
===
--- /dev/null
+++ lldb/test/API/get_darwin_real_python.py
@@ -0,0 +1,14 @@
+# On macOS, system python binaries like /usr/bin/python and $(xcrun -f python3)
+# are shims. They do some light validation work and then spawn the "real"
python
+# binary. Find the "real" python by asking dyld -- sys.executable reports the
+# wrong thing more often than not. This is also useful when we're running under
+# a Homebrew python3 binary, which also appears to be some kind of shim.
+def getDarwinRealPythonExecutable():
+import ctypes
+dyld = ctypes.cdll.LoadLibrary('/usr/lib/system/libdyld.dylib')
+namelen = ctypes.c_ulong(1024)
+name = ctypes.create_string_buffer(b'\000', namelen.value)
+dyld._NSGetExecutablePath(ctypes.byref(name), ctypes.byref(namelen))
+return name.value.decode('utf-8').strip()
+
+print(getDarwinRealPythonExecutable())
Index: lldb/test/API/lldbtest.py
===
--- lldb/test/API/lldbtest.py
+++ lldb/test/API/lldbtest.py
@@ -75,19 +75,17 @@
builddir = getBuildDir(cmd)
mkdir_p(builddir)
-# The macOS system integrity protection (SIP) doesn't allow injecting
-# libraries into system binaries, but this can be worked around by
-# copying the binary into a different location.
-if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \
-(executable.startswith('/System/') or \
-executable.startswith('/usr/bin/')):
+# On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim
+# python binary as the ASan interceptors get loaded too late. Find the
+# "real" python binary, copy it, and invoke it.
+if 'DYLD_INSERT_LIBRARIES' in test.config.environment:
copied_python = os.path.join(builddir, 'copied-system-python')
if not os.path.isfile(copied_python):
import shutil, subprocess
python = subprocess.check_output([
executable,
-'-c',
-'import sys; print(sys.executable)'
+os.path.join(os.path.dirname(os.path.realpath(__file__)),
+'get_darwin_real_python.py')
]).decode('utf-8').strip()
shutil.copy(python, copied_python)
cmd[0] = copied_python
Index: lldb/test/API/get_darwin_real_python.py
===
--- /dev/null
+++ lldb/test/API/get_darwin_real_python.py
@@ -0,0 +1,14 @@
+# On macOS, system python binaries like /usr/bin/python and $(xcrun -f python3)
+# are shims. They do some light validation work and then spawn the "real" python
+# binary. Find the "real" python by asking dyld -- sys.executable reports the
+# wrong thing more often than not. This is also useful when we're running under
+# a Homebrew python3 binary, which also appears to be some kind of shim.
+def getDarwinRealPythonExecutable():
+import
[Lldb-commits] [PATCH] D79607: [lldb/test][Darwin] Ask dyld where the real python is
JDevlieghere added a comment. Thanks Vedant for coming up with a structural solution to this problem! Comment at: lldb/test/API/lldbtest.py:36 +# a Homebrew python3 binary, which also appears to be some kind of shim. +def getDarwinRealPythonExecutable(): +import ctypes This would have to run under the Python in `executable`, not the one lit is running with. Repository: rG LLVM Github Monorepo CHANGES SINCE LAST ACTION https://reviews.llvm.org/D79607/new/ https://reviews.llvm.org/D79607 ___ lldb-commits mailing list lldb-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
[Lldb-commits] [PATCH] D79607: [lldb/test][Darwin] Ask dyld where the real python is
vsk created this revision.
vsk added a reviewer: JDevlieghere.
Herald added a project: LLDB.
On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim
python binary as the ASan interceptors get loaded too late. Find the
"real" python binary, copy it, and invoke it.
Hopefully this makes the GreenDragon and swift-ci sanitizer bots
happy...
I tested this out by running `../llvm-macosx-x86_64/bin/llvm-lit test
--filter TestNSDictionarySynthetic.py` in an ASanified swift-lldb build
directory and it worked (i.e. no more "interceptors loaded too late"
messages).
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D79607
Files:
lldb/test/API/lldbtest.py
Index: lldb/test/API/lldbtest.py
===
--- lldb/test/API/lldbtest.py
+++ lldb/test/API/lldbtest.py
@@ -28,6 +28,19 @@
if not os.path.isdir(path):
raise OSError(errno.ENOTDIR, "%s is not a directory"%path)
+# On macOS, system python binaries like /usr/bin/python and $(xcrun -f python3)
+# are shims. They do some light validation work and then spawn the "real"
python
+# binary. Find the "real" python by asking dyld -- sys.executable reports the
+# wrong thing more often than not. This is also useful when we're running under
+# a Homebrew python3 binary, which also appears to be some kind of shim.
+def getDarwinRealPythonExecutable():
+import ctypes
+dyld = ctypes.cdll.LoadLibrary('/usr/lib/system/libdyld.dylib')
+namelen = ctypes.c_ulong(1024)
+name = ctypes.create_string_buffer(b'\000', namelen.value)
+dyld._NSGetExecutablePath(ctypes.byref(name), ctypes.byref(namelen))
+return name.value.decode('utf-8').strip()
+
class LLDBTest(TestFormat):
def __init__(self, dotest_cmd):
self.dotest_cmd = dotest_cmd
@@ -75,20 +88,14 @@
builddir = getBuildDir(cmd)
mkdir_p(builddir)
-# The macOS system integrity protection (SIP) doesn't allow injecting
-# libraries into system binaries, but this can be worked around by
-# copying the binary into a different location.
-if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \
-(executable.startswith('/System/') or \
-executable.startswith('/usr/bin/')):
+# On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim
+# python binary as the ASan interceptors get loaded too late. Find the
+# "real" python binary, copy it, and invoke it.
+if 'DYLD_INSERT_LIBRARIES' in test.config.environment:
copied_python = os.path.join(builddir, 'copied-system-python')
if not os.path.isfile(copied_python):
-import shutil, subprocess
-python = subprocess.check_output([
-executable,
-'-c',
-'import sys; print(sys.executable)'
-]).decode('utf-8').strip()
+import shutil
+python = getDarwinRealPythonExecutable()
shutil.copy(python, copied_python)
cmd[0] = copied_python
Index: lldb/test/API/lldbtest.py
===
--- lldb/test/API/lldbtest.py
+++ lldb/test/API/lldbtest.py
@@ -28,6 +28,19 @@
if not os.path.isdir(path):
raise OSError(errno.ENOTDIR, "%s is not a directory"%path)
+# On macOS, system python binaries like /usr/bin/python and $(xcrun -f python3)
+# are shims. They do some light validation work and then spawn the "real" python
+# binary. Find the "real" python by asking dyld -- sys.executable reports the
+# wrong thing more often than not. This is also useful when we're running under
+# a Homebrew python3 binary, which also appears to be some kind of shim.
+def getDarwinRealPythonExecutable():
+import ctypes
+dyld = ctypes.cdll.LoadLibrary('/usr/lib/system/libdyld.dylib')
+namelen = ctypes.c_ulong(1024)
+name = ctypes.create_string_buffer(b'\000', namelen.value)
+dyld._NSGetExecutablePath(ctypes.byref(name), ctypes.byref(namelen))
+return name.value.decode('utf-8').strip()
+
class LLDBTest(TestFormat):
def __init__(self, dotest_cmd):
self.dotest_cmd = dotest_cmd
@@ -75,20 +88,14 @@
builddir = getBuildDir(cmd)
mkdir_p(builddir)
-# The macOS system integrity protection (SIP) doesn't allow injecting
-# libraries into system binaries, but this can be worked around by
-# copying the binary into a different location.
-if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \
-(executable.startswith('/System/') or \
-executable.startswith('/usr/bin/')):
+# On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim
+# python binary as the ASan interceptors get loaded too late. Find the
+# "real" python binary, copy it, and invoke it.
+if 'DYLD_INSERT_LIBRARIES'
