Author: Philip Jenvey <pjen...@underboss.org> Branch: py3k Changeset: r63600:49003927f2cc Date: 2013-04-24 18:03 -0700 http://bitbucket.org/pypy/pypy/changeset/49003927f2cc/
Log: merge default diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -35,8 +35,8 @@ "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array", "binascii", "_multiprocessing", '_warnings', "_collections", "_multibytecodec", "_ffi", - "_continuation", "_csv", "_cffi_backend", # "micronumpy", - "_posixsubprocess", + "_continuation", "_csv", "_cffi_backend", + "_posixsubprocess", # "cppyy", "micronumpy", ] )) @@ -67,6 +67,8 @@ del working_modules["_minimal_curses"] del working_modules["_posixsubprocess"] +# del working_modules["cppyy"] # not tested on win32 + # The _locale module is needed by site.py on Windows default_modules["_locale"] = None @@ -78,7 +80,7 @@ del working_modules["_minimal_curses"] del working_modules["termios"] del working_modules["_multiprocessing"] # depends on rctime - +# del working_modules["cppyy"] # depends on ctypes module_dependencies = { diff --git a/pypy/module/cppyy/capi/loadable_capi.py b/pypy/module/cppyy/capi/loadable_capi.py --- a/pypy/module/cppyy/capi/loadable_capi.py +++ b/pypy/module/cppyy/capi/loadable_capi.py @@ -11,7 +11,7 @@ C_METHOD, C_INDEX, C_INDEX_ARRAY, WLAVC_INDEX, C_METHPTRGETTER_PTR -reflection_library = 'rflxlib.so' +reflection_library = 'libcppyy_backend.so' def identify(): return 'loadable_capi' @@ -231,7 +231,7 @@ except Exception: if objectmodel.we_are_translated(): raise OperationError(space.w_ImportError, - space.wrap("missing reflection library rflxlib.so")) + space.wrap("missing reflection library %s" % reflection_library)) return False return True diff --git a/pypy/module/cppyy/test/Makefile b/pypy/module/cppyy/test/Makefile --- a/pypy/module/cppyy/test/Makefile +++ b/pypy/module/cppyy/test/Makefile @@ -3,7 +3,11 @@ std_streamsDict.so iotypesDict.so all : $(dicts) -ROOTSYS := ${ROOTSYS} +ifneq (${REFLEXHOME},) + ROOTSYS := ${REFLEXHOME} +else + ROOTSYS := ${ROOTSYS} +endif ifeq ($(ROOTSYS),) genreflex=genreflex diff --git a/pypy/tool/pypyjit.py b/pypy/tool/pypyjit.py --- a/pypy/tool/pypyjit.py +++ b/pypy/tool/pypyjit.py @@ -41,7 +41,6 @@ config.objspace.usemodules._lsprof = False # config.objspace.usemodules._ffi = True -#config.objspace.usemodules.cppyy = True config.objspace.usemodules.micronumpy = False # set_pypy_opt_level(config, level='jit') diff --git a/rpython/jit/backend/detect_cpu.py b/rpython/jit/backend/detect_cpu.py --- a/rpython/jit/backend/detect_cpu.py +++ b/rpython/jit/backend/detect_cpu.py @@ -15,7 +15,6 @@ mapping = { ('x86', '64'): [ '__amd64__', '__amd64', '__x86_64__', '__x86_64', # AMD64 - '__ia64__', '_IA64', '__IA64__' # Intel Itanium (IA-64) ], ('arm', '32'): ['__arm__', '__thumb__'], ('x86', '32'): ['i386', '__i386', '__i386__', '__i686__',], diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -1566,6 +1566,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib import rwin32 CP_ACP = 0 + BOOLP = lltype.Ptr(lltype.Array(rwin32.BOOL, hints={'nolength': True})) MultiByteToWideChar = rffi.llexternal('MultiByteToWideChar', [rffi.UINT, rwin32.DWORD, @@ -1578,7 +1579,7 @@ [rffi.UINT, rwin32.DWORD, rffi.CWCHARP, rffi.INT, rwin32.LPCSTR, rffi.INT, - rwin32.LPCSTR, rffi.VOIDP], + rwin32.LPCSTR, BOOLP], rffi.INT, calling_conv='win') @@ -1586,66 +1587,100 @@ # XXX don't know how to test this return False - def str_decode_mbcs(s, size, errors, final=False, - errorhandler=None): + def _decode_mbcs_error(s, errorhandler): + if rwin32.GetLastError() == rwin32.ERROR_NO_UNICODE_TRANSLATION: + msg = ("No mapping for the Unicode character exists in the target " + "multi-byte code page.") + errorhandler('strict', 'mbcs', msg, s, 0, 0) + else: + raise rwin32.lastWindowsError() + + def str_decode_mbcs(s, size, errors, final=False, errorhandler=None, + force_ignore=True): + if errorhandler is None: + errorhandler = default_unicode_error_decode + + if not force_ignore and errors not in ('strict', 'ignore'): + msg = "mbcs encoding does not support errors='%s'" % errors + errorhandler('strict', 'mbcs', msg, s, 0, 0) + if size == 0: return u"", 0 - if errorhandler is None: - errorhandler = default_unicode_error_decode + if force_ignore or errors == 'ignore': + flags = 0 + else: + # strict + flags = rwin32.MB_ERR_INVALID_CHARS # Skip trailing lead-byte unless 'final' is set if not final and is_dbcs_lead_byte(s[size-1]): size -= 1 - dataptr = rffi.get_nonmovingbuffer(s) - try: + with rffi.scoped_nonmovingbuffer(s) as dataptr: # first get the size of the result - usize = MultiByteToWideChar(CP_ACP, 0, + usize = MultiByteToWideChar(CP_ACP, flags, dataptr, size, lltype.nullptr(rffi.CWCHARP.TO), 0) if usize == 0: - raise rwin32.lastWindowsError() + _decode_mbcs_error(s, errorhandler) - raw_buf, gc_buf = rffi.alloc_unicodebuffer(usize) - try: + with rffi.scoped_alloc_unicodebuffer(usize) as buf: # do the conversion - if MultiByteToWideChar(CP_ACP, 0, - dataptr, size, raw_buf, usize) == 0: - raise rwin32.lastWindowsError() + if MultiByteToWideChar(CP_ACP, flags, + dataptr, size, buf.raw, usize) == 0: + _decode_mbcs_error(s, errorhandler) + return buf.str(usize), size - return (rffi.unicode_from_buffer(raw_buf, gc_buf, usize, usize), - size) - finally: - rffi.keep_unicodebuffer_alive_until_here(raw_buf, gc_buf) - finally: - rffi.free_nonmovingbuffer(s, dataptr) + def unicode_encode_mbcs(s, size, errors, errorhandler=None, + force_replace=True): + if errorhandler is None: + errorhandler = default_unicode_error_encode - def unicode_encode_mbcs(p, size, errors, errorhandler=None): + if not force_replace and errors not in ('strict', 'replace'): + msg = "mbcs encoding does not support errors='%s'" % errors + errorhandler('strict', 'mbcs', msg, s, 0, 0) + if size == 0: return '' - dataptr = rffi.get_nonmoving_unicodebuffer(p) + + if force_replace or errors == 'replace': + flags = 0 + used_default_p = lltype.nullptr(BOOLP.TO) + else: + # strict + flags = rwin32.WC_NO_BEST_FIT_CHARS + used_default_p = lltype.malloc(BOOLP.TO, 1, flavor='raw') + used_default_p[0] = rffi.cast(rwin32.BOOL, False) + try: - # first get the size of the result - mbcssize = WideCharToMultiByte(CP_ACP, 0, - dataptr, size, None, 0, - None, None) - if mbcssize == 0: - raise rwin32.lastWindowsError() + with rffi.scoped_nonmoving_unicodebuffer(s) as dataptr: + # first get the size of the result + mbcssize = WideCharToMultiByte(CP_ACP, flags, + dataptr, size, None, 0, + None, used_default_p) + if mbcssize == 0: + raise rwin32.lastWindowsError() + # If we used a default char, then we failed! + if (used_default_p and + rffi.cast(lltype.Bool, used_default_p[0])): + errorhandler('strict', 'mbcs', "invalid character", + s, 0, 0) - raw_buf, gc_buf = rffi.alloc_buffer(mbcssize) - try: - # do the conversion - if WideCharToMultiByte(CP_ACP, 0, - dataptr, size, raw_buf, mbcssize, - None, None) == 0: - raise rwin32.lastWindowsError() - - return rffi.str_from_buffer(raw_buf, gc_buf, mbcssize, mbcssize) - finally: - rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) + with rffi.scoped_alloc_buffer(mbcssize) as buf: + # do the conversion + if WideCharToMultiByte(CP_ACP, flags, + dataptr, size, buf.raw, mbcssize, + None, used_default_p) == 0: + raise rwin32.lastWindowsError() + if (used_default_p and + rffi.cast(lltype.Bool, used_default_p[0])): + errorhandler('strict', 'mbcs', "invalid character", + s, 0, 0) + return buf.str(mbcssize) finally: - rffi.free_nonmoving_unicodebuffer(p, dataptr) + if used_default_p: + lltype.free(used_default_p, flavor='raw') # ____________________________________________________________ # Decimal Encoder diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -91,6 +91,8 @@ PROCESS_VM_OPERATION PROCESS_VM_READ PROCESS_VM_WRITE CTRL_C_EVENT CTRL_BREAK_EVENT + MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION + WC_NO_BEST_FIT_CHARS """ from rpython.translator.platform import host_factory static_platform = host_factory() diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py --- a/rpython/rlib/test/test_runicode.py +++ b/rpython/rlib/test/test_runicode.py @@ -728,6 +728,30 @@ self.checkencode(u"\N{GREEK CAPITAL LETTER PHI}", "mbcs") # a F self.checkencode(u"\N{GREEK CAPITAL LETTER PSI}", "mbcs") # a ? + def test_mbcs_decode_force_ignore(self): + if sys.platform != 'win32': + py.test.skip("mbcs encoding is win32-specific") + + # XXX: requires a locale w/ a restrictive encoding to test + from rpython.rlib.rlocale import getdefaultlocale + if getdefaultlocale()[1] != 'cp932': + py.test.skip("requires cp932 locale") + + s = '\xff\xf4\x8f\xbf\xbf' + encoder = self.getdecoder('mbcs') + assert encoder(s, len(s), 'strict') == (u'\U0010ffff', 5) + py.test.raises(UnicodeEncodeError, encoder, s, len(s), 'strict', + force_ignore=False) + + def test_mbcs_encode_force_replace(self): + if sys.platform != 'win32': + py.test.skip("mbcs encoding is win32-specific") + u = u'@test_2224_tmp-?L??\udc80' + encoder = self.getencoder('mbcs') + assert encoder(u, len(u), 'strict') == '@test_2224_tmp-?L???' + py.test.raises(UnicodeEncodeError, encoder, u, len(u), 'strict', + force_replace=False) + def test_encode_decimal(self): encoder = self.getencoder('decimal') assert encoder(u' 12, 34 ', 8, None) == ' 12, 34 ' diff --git a/rpython/translator/platform/bsd.py b/rpython/translator/platform/bsd.py new file mode 100644 --- /dev/null +++ b/rpython/translator/platform/bsd.py @@ -0,0 +1,31 @@ + +import os +from rpython.translator.platform import posix + +class BSD(posix.BasePosix): + DEFAULT_CC = 'clang' + + so_ext = 'so' + make_cmd = 'gmake' + + standalone_only = [] + shared_only = [] + + def _args_for_shared(self, args): + return ['-shared'] + args + + def _include_dirs_for_libffi(self): + return [os.path.join(os.environ.get("LOCALBASE", "/usr/local"), "include")] + + def _library_dirs_for_libffi(self): + return [os.path.join(os.environ.get("LOCALBASE", "/usr/local"), "lib")] + + def _preprocess_include_dirs(self, include_dirs): + res_incl_dirs = list(include_dirs) + res_incl_dirs.append(os.path.join(os.environ.get("LOCALBASE", "/usr/local"), "include")) + return res_incl_dirs + + def _preprocess_library_dirs(self, library_dirs): + res_lib_dirs = list(library_dirs) + res_lib_dirs.append(os.path.join(os.environ.get("LOCALBASE", "/usr/local"), "lib")) + return res_lib_dirs diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -9,11 +9,7 @@ shared_only = () so_ext = 'dylib' - - # NOTE: With asmgcc GCC 4.2 will fail at runtime due to subtle issues, - # possibly related to GC roots. Using LLVM-GCC or Clang will break the - # build. On Darwin asmgcc is not the default anymore, so it is fine to use - # whatever gcc we find on the system + DEFAULT_CC = 'clang' def _args_for_shared(self, args): return (list(self.shared_only) @@ -26,10 +22,6 @@ def _library_dirs_for_libffi(self): return ['/usr/lib'] - def check___thread(self): - # currently __thread is not supported by Darwin gccs - return False - def _frameworks(self, frameworks): args = [] for f in frameworks: diff --git a/rpython/translator/platform/freebsd.py b/rpython/translator/platform/freebsd.py --- a/rpython/translator/platform/freebsd.py +++ b/rpython/translator/platform/freebsd.py @@ -1,60 +1,14 @@ """Support for FreeBSD.""" import os +from rpython.translator.platform.bsd import BSD -from rpython.translator.platform import posix - -def get_env(key, default): - if key in os.environ: - return os.environ[key] - else: - return default - -def get_env_vector(key, default): - string = get_env(key, default) - # XXX: handle quotes - return string.split() - -class Freebsd(posix.BasePosix): +class Freebsd(BSD): name = "freebsd" - link_flags = ['-pthread'] + get_env_vector('LDFLAGS', '') + link_flags = ['-pthread'] + os.environ.get('LDFLAGS', '').split() cflags = ['-O3', '-pthread', '-fomit-frame-pointer' - ] + get_env_vector('CFLAGS', '') - standalone_only = [] - shared_only = [] - so_ext = 'so' - make_cmd = 'gmake' - - def __init__(self, cc=None): - if cc is None: - cc = get_env("CC", "gcc") - super(Freebsd, self).__init__(cc) - - def _args_for_shared(self, args): - return ['-shared'] + args - - def _preprocess_include_dirs(self, include_dirs): - res_incl_dirs = list(include_dirs) - res_incl_dirs.append(os.path.join(get_env("LOCALBASE", "/usr/local"), "include")) - return res_incl_dirs - - def _preprocess_library_dirs(self, library_dirs): - res_lib_dirs = list(library_dirs) - res_lib_dirs.append(os.path.join(get_env("LOCALBASE", "/usr/local"), "lib")) - return res_lib_dirs - - def _include_dirs_for_libffi(self): - return [os.path.join(get_env("LOCALBASE", "/usr/local"), "include")] - - def _library_dirs_for_libffi(self): - return [os.path.join(get_env("LOCALBASE", "/usr/local"), "lib")] + ] + os.environ.get('CFLAGS', '').split() class Freebsd_64(Freebsd): shared_only = ('-fPIC',) - -class GNUkFreebsd(Freebsd): - extra_libs = ('-lrt',) - -class GNUkFreebsd_64(Freebsd_64): - extra_libs = ('-lrt',) diff --git a/rpython/translator/platform/openbsd.py b/rpython/translator/platform/openbsd.py --- a/rpython/translator/platform/openbsd.py +++ b/rpython/translator/platform/openbsd.py @@ -2,60 +2,17 @@ import os -from rpython.translator.platform import posix +from pypy.translator.platform.bsd import BSD -def get_env(key, default): - if key in os.environ: - return os.environ[key] - else: - return default - -def get_env_vector(key, default): - string = get_env(key, default) - # XXX: handle quotes - return string.split() - -class OpenBSD(posix.BasePosix): +class OpenBSD(BSD): name = "openbsd" - link_flags = get_env_vector("LDFLAGS", '-pthread') - cflags = get_env_vector("CFLAGS", "-O3 -pthread -fomit-frame-pointer -D_BSD_SOURCE") - standalone_only = [] - shared_only = [] - so_ext = 'so' - make_cmd = 'gmake' - - def __init__(self, cc=None): - if cc is None: - cc = get_env("CC", "gcc") - super(OpenBSD, self).__init__(cc) - - def _args_for_shared(self, args): - return ['-shared'] + args - - def _preprocess_include_dirs(self, include_dirs): - res_incl_dirs = list(include_dirs) - res_incl_dirs.append(os.path.join(get_env("LOCALBASE", "/usr/local"), "include")) - return res_incl_dirs - - def _preprocess_library_dirs(self, library_dirs): - res_lib_dirs = list(library_dirs) - res_lib_dirs.append(os.path.join(get_env("LOCALBASE", "/usr/local"), "lib")) - return res_lib_dirs - - def _include_dirs_for_libffi(self): - return [os.path.join(get_env("LOCALBASE", "/usr/local"), "include")] - - def _library_dirs_for_libffi(self): - return [os.path.join(get_env("LOCALBASE", "/usr/local"), "lib")] + link_flags = os.environ.get("LDFLAGS", '-pthread').split() + cflags = os.environ.get("CFLAGS", "-O3 -pthread -fomit-frame-pointer -D_BSD_SOURCE").split() def _libs(self, libraries): libraries=set(libraries + ("intl", "iconv", "compat")) return ['-l%s' % lib for lib in libraries if lib not in ["crypt", "dl", "rt"]] - def check___thread(self): - # currently __thread is not supported by Darwin gccs - return False - class OpenBSD_64(OpenBSD): shared_only = ('-fPIC',) diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -13,13 +13,10 @@ relevant_environ = ('CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH') + DEFAULT_CC = 'gcc' + def __init__(self, cc=None): - if cc is None: - try: - cc = os.environ['CC'] - except KeyError: - cc = 'gcc' - self.cc = cc + self.cc = cc or os.environ.get('CC', self.DEFAULT_CC) def _libs(self, libraries): return ['-l%s' % lib for lib in libraries] @@ -250,7 +247,7 @@ self.defs = {} self.lines = [] self.makefile_dir = py.path.local(path) - + def pathrel(self, fpath): if fpath.dirpath() == self.makefile_dir: return fpath.basename _______________________________________________ pypy-commit mailing list pypy-commit@python.org http://mail.python.org/mailman/listinfo/pypy-commit