Giovanni Bajo wrote:
> On sab, 2009-02-07 at 00:17 +0100, Lorenzo Mancini wrote:
>> Index: deploy/tools/PyInstaller/bindepend.py
>> ===================================================================
>> --- deploy/tools/PyInstaller/bindepend.py (revision 21414)
>> +++ deploy/tools/PyInstaller/bindepend.py (working copy)
>> @@ -354,7 +354,12 @@
>> m = re.search(r"\s+(.*?)\s+\(.*\)", line)
>> if m:
>> lib = m.group(1)
>> - if os.path.exists(lib):
>> + # If a dylib lists itself as dependency (?), using its bare
>> filename
>> + # instead of full path, it won't probably be found as is on the
>> + # filesystem; we skip the check.
>> + if lib == os.path.basename(pth):
>> + continue
>> + elif os.path.exists(lib):
>> rslt.append(lib)
>> else:
>> print 'E: cannot find path %s (needed by %s)' % \
>
> The comment needs rewording because I can't understand what the problem
> is.
I removed this hunk in the new version of the patch. The rationale
behind it was that a typical dylib uses to have an explicit dependency
upon itself, in addition to "real" dependencies upon other shared
libraries. For instance:
$ otool -L libncurses.5.dylib
libncurses.5.dylib:
/Library/Frameworks/Python.framework/Versions/2.5/lib/libncurses.5.dylib
(compatibility version 5.0.0, current version 5.0.0)
[...]
From what I've gathered, this is the "id" of the dylib, and seems to be
used to store the path the library is installed to, so that binaries
linked against that library will record that path as the one dyld should
use to locate it.
Now, I was developing the patch using a dylib which had a bare filename
(no leading path) as id; when PyInstaller began to crash because it
wasn't finding all its dependencies (again, because the id of the
problematic dylib was expressed with a bare filename), I added the check.
But now, after replacing the bare filename with the full install path as
the dylib's id, I realized that the existing PyInstaller machinery is
already able to collect the full set of dependencies; every dylib will
be listed twice, but function Dependencies (see bindepend.py) already
throws away duplicates. That's why I removed that hunk.
>> Index: deploy/tools/PyInstaller/Build.py
> The comment needs rewording because I can't understand what that code is
> about.
Done, please take a look.
>> Index: deploy/tools/PyInstaller/mf.py
>> ===================================================================
>> --- deploy/tools/PyInstaller/mf.py (revision 21414)
>> +++ deploy/tools/PyInstaller/mf.py (working copy)
>> @@ -25,6 +25,12 @@
>> except ImportError:
>> zipimport = None
>>
>> +try:
>> + # ctypes is supported starting with python 2.5
>> + import ctypes
>> +except ImportError:
> I would say "if ctypes is present, we can enable specific dependency
> discovery". Python 2.5 is when it shipped through the standard library,
> but it's been available before as a 3rd-party extension and your code
> would work in that case just as well.
Right, I overlooked that, thanks.
>> @@ -508,6 +514,45 @@
>> def ispackage(self, nm):
>> return self.modules[nm].ispackage()
>>
>> + def _resolveCtypesImports(self, mod):
>> + """Completes ctypes BINARY entries intended in module with their
>> + full path.
> I wouldn't fully replace it: just prepend the user-specified paths.
Done.
>> + for cbin in cbinaries:
>> +
>> + # Executes find_library using the local paths of ImportTracker
>> as
>> + # library search paths, then replaces original values.
>> +
>> + old = _savePaths()
>> + from ctypes.util import find_library
>> + cpath = find_library(os.path.splitext(cbin)[0])
>> + _restorePaths(old)
> Given that you're doing nothing here but calling find_library, I think
> it's safe to call savePaths() / restorePaths() just once (before and
> after the for loop).
Done.
>
>> @@ -536,6 +581,10 @@
>> # or
>> # mod = director.getmod(nm)
>> if mod:
>> +
>> + if ctypes and isinstance(mod, PyModule):
>> + self._resolveCtypesImports(mod)
>> +
>> mod.__name__ = fqname
>> self.modules[fqname] = mod
>> # now look for hooks
>
> Please remove those extra newlines.
Done. I'm attaching the updated patch.
> Amazing work, thanks! The only thing missing is some tests, and
> documentation. Can you please fill in the empty wiki page linked from
> here?
I'm attaching a patch adding a test for the feature too (tested and
working under Mac OS X only).
> http://pyinstaller.python-hosting.com/wiki/SupportedPackages
I think I'm not allowed to; I'm not seeing any "Edit this page" button,
even when logged in.
--
Lorenzo Mancini
--~--~---------~--~----~------------~-------~--~----~
You received this message because you are subscribed to the Google Groups
"PyInstaller" group.
To post to this group, send email to [email protected]
To unsubscribe from this group, send email to
[email protected]
For more options, visit this group at
http://groups.google.com/group/PyInstaller?hl=en
-~----------~----~----~----~------~----~------~--~---
Index: Build.py
===================================================================
--- Build.py (revision 21448)
+++ Build.py (working copy)
@@ -317,10 +317,15 @@
elif isinstance(mod, mf.PkgInZipModule):
zipfiles.append((os.path.basename(str(mod.owner)),
str(mod.owner), 'ZIPFILE'))
- elif modnm == '__main__':
- pass
else:
- pure.append((modnm, fnm, 'PYMODULE'))
+ # mf.PyModule instances expose a list of binary
+ # dependencies, most probably shared libraries accessed
+ # via ctypes. Add them to the overall required
binaries.
+ binaries.extend(mod.binaries)
+ if modnm == '__main__':
+ pass
+ else:
+ pure.append((modnm, fnm, 'PYMODULE'))
binaries.extend(bindepend.Dependencies(binaries,
platform=target_platform))
self.fixMissingPythonLib(binaries)
Index: mf.py
===================================================================
--- mf.py (revision 21448)
+++ mf.py (working copy)
@@ -25,6 +25,13 @@
except ImportError:
zipimport = None
+try:
+ # if ctypes is present, we can enable specific dependency discovery
+ import ctypes
+ from ctypes.util import find_library
+except ImportError:
+ ctypes = None
+
import suffixes
try:
@@ -78,7 +85,7 @@
def _getsuffixes(self):
return suffixes.get_suffixes(self.target_platform)
-
+
def getmod(self, nm, getsuffixes=None, loadco=marshal.loads):
if getsuffixes is None:
getsuffixes = self._getsuffixes
@@ -508,6 +515,41 @@
def ispackage(self, nm):
return self.modules[nm].ispackage()
+ def _resolveCtypesImports(self, mod):
+ """Completes ctypes BINARY entries for modules with their full path.
+ """
+ if sys.platform.startswith("linux"):
+ envvar = "LD_LIBRARY_PATH"
+ elif sys.platform.startswith("darwin"):
+ envvar = "DYLD_LIBRARY_PATH"
+ else:
+ envvar = "PATH"
+
+ def _savePaths():
+ old = os.environ.get(envvar, None)
+ os.environ[envvar] = os.pathsep.join(self.path)
+ if old is not None:
+ os.environ[envvar] = os.pathsep.join([os.environ[envvar], old])
+ return old
+
+ def _restorePaths(old):
+ del os.environ[envvar]
+ if old is not None:
+ os.environ[envvar] = old
+
+ cbinaries = list(mod.binaries)
+ mod.binaries = []
+ # Executes find_library prepending ImportTracker's local paths to
+ # library search paths, then replaces original values.
+ old = _savePaths()
+ for cbin in cbinaries:
+ cpath = find_library(os.path.splitext(cbin)[0])
+ if cpath is None:
+ print "W: library %s required via ctypes not found" % (cbin,)
+ else:
+ mod.binaries.append((cbin, cpath, "BINARY"))
+ _restorePaths(old)
+
def doimport(self, nm, ctx, fqname):
# Not that nm is NEVER a dotted name at this point
assert ("." not in nm), nm
@@ -536,6 +578,8 @@
# or
# mod = director.getmod(nm)
if mod:
+ if ctypes and isinstance(mod, PyModule):
+ self._resolveCtypesImports(mod)
mod.__name__ = fqname
self.modules[fqname] = mod
# now look for hooks
@@ -602,6 +646,7 @@
self._all = []
self.imports = []
self.warnings = []
+ self.binaries = []
self._xref = {}
def ispackage(self):
@@ -614,7 +659,7 @@
self._xref[nm] = 1
def __str__(self):
- return "<Module %s %s %s>" % (self.__name__, self.__file__,
self.imports)
+ return "<Module %s %s %s %s>" % (self.__name__, self.__file__,
self.imports, self.binaries)
class BuiltinModule(Module):
typ = 'BUILTIN'
@@ -641,7 +686,7 @@
self.scancode()
def scancode(self):
- self.imports, self.warnings, allnms = scan_code(self.co)
+ self.imports, self.warnings, self.binaries, allnms = scan_code(self.co)
if allnms:
self._all = allnms
@@ -715,6 +760,8 @@
STORE_FAST = dis.opname.index('STORE_FAST')
STORE_GLOBAL = dis.opname.index('STORE_GLOBAL')
LOAD_GLOBAL = dis.opname.index('LOAD_GLOBAL')
+LOAD_ATTR = dis.opname.index('LOAD_ATTR')
+LOAD_NAME = dis.opname.index('LOAD_NAME')
EXEC_STMT = dis.opname.index('EXEC_STMT')
try:
SET_LINENO = dis.opname.index('SET_LINENO')
@@ -767,12 +814,14 @@
instrs.append((op, oparg, incondition, curline))
return instrs
-def scan_code(co, m=None, w=None, nested=0):
+def scan_code(co, m=None, w=None, b=None, nested=0):
instrs = pass1(co.co_code)
if m is None:
m = []
if w is None:
w = []
+ if b is None:
+ b = []
all = None
lastname = None
level = -1 # import-level, same behaviour as up to Python 2.4
@@ -833,7 +882,99 @@
w.append("W: %s %s exec statement detected at line %s" % (lvl,
cndtl, curline))
else:
lastname = None
+
+ if ctypes:
+ # ctypes scanning requires a scope wider than one bytecode
instruction,
+ # so the code resides in a separate function for clarity.
+ ctypesb, ctypesw = scan_code_for_ctypes(co, instrs, i)
+ b.extend(ctypesb)
+ w.extend(ctypesw)
+
for c in co.co_consts:
if isinstance(c, type(co)):
- scan_code(c, m, w, 1)
- return m, w, all
+ # FIXME: "all" was not updated here nor returned. Was it the
desired
+ # behaviour?
+ _, _, _, all_nested = scan_code(c, m, w, b, 1)
+ if all_nested:
+ all.extend(all_nested)
+ return m, w, b, all
+
+def scan_code_for_ctypes(co, instrs, i):
+ """Detects ctypes dependencies, using reasonable heuristics that should
+ cover most common ctypes usages; returns a tuple of two lists, one
+ containing names of binaries detected as dependencies, the other containing
+ warnings.
+ """
+
+ def _libFromConst(i):
+ """Extracts library name from an expected LOAD_CONST instruction and
+ appends it to local binaries list.
+ """
+ op, oparg, conditional, curline = instrs[i]
+ if op == LOAD_CONST:
+ soname = co.co_consts[oparg]
+ b.append(soname)
+
+ b = []
+
+ op, oparg, conditional, curline = instrs[i]
+
+ if op in (LOAD_GLOBAL, LOAD_NAME):
+ name = co.co_names[oparg]
+
+ if name in ("CDLL", "WinDLL"):
+ # Guesses ctypes imports of this type: CDLL("library.so")
+
+ # LOAD_GLOBAL 0 (CDLL) <--- we "are" here right now
+ # LOAD_CONST 1 ('library.so')
+
+ _libFromConst(i+1)
+
+ elif name == "ctypes":
+ # Guesses ctypes imports of this type: ctypes.DLL("library.so")
+
+ # LOAD_GLOBAL 0 (ctypes) <--- we "are" here right now
+ # LOAD_ATTR 1 (CDLL)
+ # LOAD_CONST 1 ('library.so')
+
+ op2, oparg2, conditional2, curline2 = instrs[i+1]
+ if op2 == LOAD_ATTR:
+ if co.co_names[oparg2] in ("CDLL", "WinDLL"):
+ # Fetch next, and finally get the library name
+ _libFromConst(i+2)
+
+ elif name == ("cdll", "windll"):
+ # Guesses ctypes imports of these types:
+
+ # * cdll.library (only valid on Windows)
+
+ # LOAD_GLOBAL 0 (cdll) <--- we "are" here right now
+ # LOAD_ATTR 1 (library)
+
+ # * cdll.LoadLibrary("library.so")
+
+ # LOAD_GLOBAL 0 (cdll) <--- we "are" here right
now
+ # LOAD_ATTR 1 (LoadLibrary)
+ # LOAD_CONST 1 ('library.so')
+
+ op2, oparg2, conditional2, curline2 = instrs[i+1]
+ if op2 == LOAD_ATTR:
+ if co.co_names[oparg2] != "LoadLibrary":
+ # First type
+ soname = co.co_names[oparg2] + ".dll"
+ b.append(soname)
+ else:
+ # Second type, needs to fetch one more instruction
+ _libFromConst(i+2)
+
+ # If any of the libraries has been requested with anything different from
+ # the bare filename, drop that entry and warn the user - pyinstaller would
+ # need to patch the compiled pyc file to make it work correctly!
+
+ w = []
+ for bin in list(b):
+ if bin != os.path.basename(bin):
+ b.remove(bin)
+ w.append("W: ignoring %s - ctypes imports only supported using
bare filenames" % (bin,))
+
+ return b, w
Index: buildtests/ctypes/testctypes.c
===================================================================
--- buildtests/ctypes/testctypes.c (revision 0)
+++ buildtests/ctypes/testctypes.c (revision 0)
@@ -0,0 +1,4 @@
+int dummy(int arg)
+{
+ return arg;
+}
Index: buildtests/test15.py
===================================================================
--- buildtests/test15.py (revision 0)
+++ buildtests/test15.py (revision 0)
@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+try:
+ import ctypes
+except ImportError:
+ # ctypes unavailable, testing ctypes support is pointless.
+ sys.exit(0)
+
+import test15a
+assert test15a.dummy(42) == 42
Index: buildtests/test15a.py
===================================================================
--- buildtests/test15a.py (revision 0)
+++ buildtests/test15a.py (revision 0)
@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+from ctypes import *
+
+def dummy(arg):
+ tct = CDLL("testctypes.dylib")
+ return tct.dummy(arg)
Index: buildtests/runtests.py
===================================================================
--- buildtests/runtests.py (revision 21448)
+++ buildtests/runtests.py (working copy)
@@ -114,9 +114,18 @@
# Run the test in a clean environment to make sure they're
# really self-contained
del os.environ["PATH"]
- prog = os.path.join('dist', test, test)
- if not os.path.exists(prog):
- prog = os.path.join(prog + '.exe')
+
+ of_prog = os.path.join('dist', test) # one-file deploy filename
+ od_prog = os.path.join('dist', test, test) # one-dir deploy
filename
+
+ if os.path.isfile(of_prog):
+ prog = of_prog
+ else:
+ if os.path.isfile(od_prog):
+ prog = od_prog
+ else:
+ prog = od_prog + ".exe"
+
print "RUNNING:", prog
res = os.system(prog)
os.environ["PATH"] = path
Index: buildtests/test15.spec
===================================================================
--- buildtests/test15.spec (revision 0)
+++ buildtests/test15.spec (revision 0)
@@ -0,0 +1,36 @@
+# -*- mode: python -*-
+
+import sys
+if not sys.platform.startswith("darwin"):
+ raise RuntimeError("please port test15 under linux2 and win32")
+
+import os
+
+# If the required dylib does not reside in the current directory, the Analysis
+# class machinery, based on ctypes.util.find_library, will not find it. This
was
+# done on purpose for this test, to show how to give Analysis class a clue.
+os.environ["DYLD_LIBRARY_PATH"] = "ctypes/"
+
+# Check for presence of testctypes shared library, build it if not present
+if not os.path.exists("ctypes/testctypes.dylib"):
+ os.chdir("ctypes")
+ os.system("gcc -Wall -dynamiclib testctypes.c -o testctypes.dylib
-headerpad_max_install_names")
+ id_dylib = os.path.abspath("testctypes.dylib")
+ os.system("install_name_tool -id %s testctypes.dylib" % (id_dylib,))
+ os.chdir("..")
+
+__testname__ = 'test15'
+
+a = Analysis(['../support/_mountzlib.py',
+ '../support/useUnicode.py',
+ 'test15.py'],
+ pathex=[])
+pyz = PYZ(a.pure)
+exe = EXE(pyz,
+ a.scripts,
+ a.binaries,
+ name=os.path.join('dist', __testname__),
+ debug=False,
+ strip=False,
+ upx=False,
+ console=1 )
