Lorenzo Mancini wrote:
> Hello,
>
> PyInstaller doesn't currently have analysis support to detect libraries
> used through ctypes.
>
> The attached patch implements a minimal support for most common uses of
> ctypes, namely:
>
> * CDLL("library.so")
> * ctypes.DLL("library.so")
> * cdll.library
> * cdll.LoadLibrary("library.so")
>
> (and their windows counterparts of course).
>
> The patch adds some bytecode magic in scan_code to detect the above uses
> of ctypes, and gives PyModule instances the ability to store binary file
> names for the referenced libraries (see mf.py). Analysis class (see
> Build.py) is finally given awareness of this new ability for PyModule
> instances, so that it can update the binaries' list accordingly.
The improved attached version:
* checks for ctypes' presence before doing anything;
* correctly resolves full pathnames for the imported libraries using
ctypes.util.find_library;
* emits warnings if the analyzed ctypes code contains a full or
relative path to the library, and drops it if so - even if we included
it in the final package, it would be required on pyinstaller's side to
patch the compiled pyc for things to work correctly.
--
Lorenzo Mancini
--~--~---------~--~----~------------~-------~--~----~
You received this message because you are subscribed to the Google Groups
"PyInstaller" group.
To post to this group, send email to [email protected]
To unsubscribe from this group, send email to
[email protected]
For more options, visit this group at
http://groups.google.com/group/PyInstaller?hl=en
-~----------~----~----~----~------~----~------~--~---
Index: deploy/tools/PyInstaller/bindepend.py
===================================================================
--- deploy/tools/PyInstaller/bindepend.py (revision 21414)
+++ deploy/tools/PyInstaller/bindepend.py (working copy)
@@ -354,7 +354,12 @@
m = re.search(r"\s+(.*?)\s+\(.*\)", line)
if m:
lib = m.group(1)
- if os.path.exists(lib):
+ # If a dylib lists itself as dependency (?), using its bare
filename
+ # instead of full path, it won't probably be found as is on the
+ # filesystem; we skip the check.
+ if lib == os.path.basename(pth):
+ continue
+ elif os.path.exists(lib):
rslt.append(lib)
else:
print 'E: cannot find path %s (needed by %s)' % \
Index: deploy/tools/PyInstaller/Build.py
===================================================================
--- deploy/tools/PyInstaller/Build.py (revision 21414)
+++ deploy/tools/PyInstaller/Build.py (working copy)
@@ -318,8 +318,12 @@
zipfiles.append((os.path.basename(str(mod.owner)),
str(mod.owner), 'ZIPFILE'))
elif modnm == '__main__':
- pass
+ # This and the next case are about mf.PyModule
+ # instances. Take care of their required shared
+ # libraries via ctypes.
+ binaries.extend(mod.binaries)
else:
+ binaries.extend(mod.binaries)
pure.append((modnm, fnm, 'PYMODULE'))
binaries.extend(bindepend.Dependencies(binaries,
platform=target_platform))
Index: deploy/tools/PyInstaller/mf.py
===================================================================
--- deploy/tools/PyInstaller/mf.py (revision 21414)
+++ deploy/tools/PyInstaller/mf.py (working copy)
@@ -25,6 +25,12 @@
except ImportError:
zipimport = None
+try:
+ # ctypes is supported starting with python 2.5
+ import ctypes
+except ImportError:
+ ctypes = None
+
import suffixes
try:
@@ -78,7 +84,7 @@
def _getsuffixes(self):
return suffixes.get_suffixes(self.target_platform)
-
+
def getmod(self, nm, getsuffixes=None, loadco=marshal.loads):
if getsuffixes is None:
getsuffixes = self._getsuffixes
@@ -508,6 +514,45 @@
def ispackage(self, nm):
return self.modules[nm].ispackage()
+ def _resolveCtypesImports(self, mod):
+ """Completes ctypes BINARY entries intended in module with their
+ full path.
+ """
+
+ if sys.platform.startswith("linux"):
+ envvar = "LD_LIBRARY_PATH"
+ elif sys.platform.startswith("darwin"):
+ envvar = "DYLD_LIBRARY_PATH"
+ else:
+ envvar = "PATH"
+
+ def _savePaths():
+ old = os.environ.get(envvar, None)
+ os.environ[envvar] = os.pathsep.join(self.path)
+ return old
+
+ def _restorePaths(old):
+ del os.environ[envvar]
+ if old:
+ os.environ[envvar] = old
+
+ cbinaries = list(mod.binaries)
+ mod.binaries = []
+ for cbin in cbinaries:
+
+ # Executes find_library using the local paths of ImportTracker as
+ # library search paths, then replaces original values.
+
+ old = _savePaths()
+ from ctypes.util import find_library
+ cpath = find_library(os.path.splitext(cbin)[0])
+ _restorePaths(old)
+
+ if cpath is None:
+ print "W: library %s required via ctypes not found" % (cbin,)
+ else:
+ mod.binaries.append((cbin, cpath, "BINARY"))
+
def doimport(self, nm, ctx, fqname):
# Not that nm is NEVER a dotted name at this point
assert ("." not in nm), nm
@@ -536,6 +581,10 @@
# or
# mod = director.getmod(nm)
if mod:
+
+ if ctypes and isinstance(mod, PyModule):
+ self._resolveCtypesImports(mod)
+
mod.__name__ = fqname
self.modules[fqname] = mod
# now look for hooks
@@ -602,6 +651,7 @@
self._all = []
self.imports = []
self.warnings = []
+ self.binaries = []
self._xref = {}
def ispackage(self):
@@ -614,7 +664,7 @@
self._xref[nm] = 1
def __str__(self):
- return "<Module %s %s %s>" % (self.__name__, self.__file__,
self.imports)
+ return "<Module %s %s %s %s>" % (self.__name__, self.__file__,
self.imports, self.binaries)
class BuiltinModule(Module):
typ = 'BUILTIN'
@@ -641,7 +691,7 @@
self.scancode()
def scancode(self):
- self.imports, self.warnings, allnms = scan_code(self.co)
+ self.imports, self.warnings, self.binaries, allnms = scan_code(self.co)
if allnms:
self._all = allnms
@@ -715,6 +765,8 @@
STORE_FAST = dis.opname.index('STORE_FAST')
STORE_GLOBAL = dis.opname.index('STORE_GLOBAL')
LOAD_GLOBAL = dis.opname.index('LOAD_GLOBAL')
+LOAD_ATTR = dis.opname.index('LOAD_ATTR')
+LOAD_NAME = dis.opname.index('LOAD_NAME')
EXEC_STMT = dis.opname.index('EXEC_STMT')
try:
SET_LINENO = dis.opname.index('SET_LINENO')
@@ -767,12 +819,14 @@
instrs.append((op, oparg, incondition, curline))
return instrs
-def scan_code(co, m=None, w=None, nested=0):
+def scan_code(co, m=None, w=None, b=None, nested=0):
instrs = pass1(co.co_code)
if m is None:
m = []
if w is None:
w = []
+ if b is None:
+ b = []
all = None
lastname = None
level = -1 # import-level, same behaviour as up to Python 2.4
@@ -833,7 +887,99 @@
w.append("W: %s %s exec statement detected at line %s" % (lvl,
cndtl, curline))
else:
lastname = None
+
+ if ctypes:
+ # ctypes scanning requires a scope wider than one bytecode
instruction,
+ # so the code resides in a separate function for clarity.
+ ctypesb, ctypesw = scan_code_for_ctypes(co, instrs, i)
+ b.extend(ctypesb)
+ w.extend(ctypesw)
+
for c in co.co_consts:
if isinstance(c, type(co)):
- scan_code(c, m, w, 1)
- return m, w, all
+ # FIXME: "all" was not updated here nor returned. Was it the
desired
+ # behaviour?
+ _, _, _, all_nested = scan_code(c, m, w, b, 1)
+ if all_nested:
+ all.extend(all_nested)
+ return m, w, b, all
+
+def scan_code_for_ctypes(co, instrs, i):
+ """Detects ctypes dependencies, using reasonable heuristics that should
+ cover most common ctypes usages; returns a tuple of two lists, one
+ containing names of binaries detected as dependencies, the other containing
+ warnings.
+ """
+
+ def _libFromConst(i):
+ """Extracts library name from an expected LOAD_CONST instruction and
+ appends it to local binaries list.
+ """
+ op, oparg, conditional, curline = instrs[i]
+ if op == LOAD_CONST:
+ soname = co.co_consts[oparg]
+ b.append(soname)
+
+ b = []
+
+ op, oparg, conditional, curline = instrs[i]
+
+ if op in (LOAD_GLOBAL, LOAD_NAME):
+ name = co.co_names[oparg]
+
+ if name in ("CDLL", "WinDLL"):
+ # Guesses ctypes imports of this type: CDLL("library.so")
+
+ # LOAD_GLOBAL 0 (CDLL) <--- we "are" here right now
+ # LOAD_CONST 1 ('library.so')
+
+ _libFromConst(i+1)
+
+ elif name == "ctypes":
+ # Guesses ctypes imports of this type: ctypes.DLL("library.so")
+
+ # LOAD_GLOBAL 0 (ctypes) <--- we "are" here right now
+ # LOAD_ATTR 1 (CDLL)
+ # LOAD_CONST 1 ('library.so')
+
+ op2, oparg2, conditional2, curline2 = instrs[i+1]
+ if op2 == LOAD_ATTR:
+ if co.co_names[oparg2] in ("CDLL", "WinDLL"):
+ # Fetch next, and finally get the library name
+ _libFromConst(i+2)
+
+ elif name == ("cdll", "windll"):
+ # Guesses ctypes imports of these types:
+
+ # * cdll.library (only valid on Windows)
+
+ # LOAD_GLOBAL 0 (cdll) <--- we "are" here right now
+ # LOAD_ATTR 1 (library)
+
+ # * cdll.LoadLibrary("library.so")
+
+ # LOAD_GLOBAL 0 (cdll) <--- we "are" here right
now
+ # LOAD_ATTR 1 (LoadLibrary)
+ # LOAD_CONST 1 ('library.so')
+
+ op2, oparg2, conditional2, curline2 = instrs[i+1]
+ if op2 == LOAD_ATTR:
+ if co.co_names[oparg2] != "LoadLibrary":
+ # First type
+ soname = co.co_names[oparg2] + ".dll"
+ b.append(soname)
+ else:
+ # Second type, needs to fetch one more instruction
+ _libFromConst(i+2)
+
+ # If any of the libraries has been requested with anything different from
+ # the bare filename, drop that entry and warn the user - pyinstaller would
+ # need to patch the compiled pyc file to make it work correctly!
+
+ w = []
+ for bin in list(b):
+ if bin != os.path.basename(bin):
+ b.remove(bin)
+ w.append("W: ignoring %s - ctypes imports only supported using
bare filenames" % (bin,))
+
+ return b, w
