Author: Matti Picus <matti.pi...@gmail.com> Branch: Changeset: r93811:496d05d4758e Date: 2018-02-12 14:06 -0500 http://bitbucket.org/pypy/pypy/changeset/496d05d4758e/
Log: merge msvc14 which allows using Visual Studio 17 compiler suite diff too long, truncating to 2000 out of 2842 lines diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -39,10 +39,24 @@ .. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266 +Installing "Build Tools for Visual Studio 2017" (for Python 3) +-------------------------------------------------------------- + +As documented in the CPython Wiki_, CPython now recommends Visual C++ version +14.0. A compact version of the compiler suite can be obtained from Microsoft_ +downloads, search the page for "Build Tools for Visual Studio 2017". + +You will also need to install the the `Windows SDK`_ in order to use the +`mt.exe` mainfest compiler. + +.. _Wiki: https://wiki.python.org/moin/WindowsCompilers +.. _Microsoft: https://www.visualstudio.com/downloads +.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk + Translating PyPy with Visual Studio ----------------------------------- -We routinely test translation using v9, also known as Visual Studio 2008. +We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -82,8 +96,8 @@ .. _build instructions: http://pypy.org/download.html#building-from-source -Setting Up Visual Studio for building SSL in Python3 ----------------------------------------------------- +Setting Up Visual Studio 9.0 for building SSL in Python3 +-------------------------------------------------------- On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after translation. However ``distutils`` does not support the Micorosft-provided Visual C @@ -132,243 +146,14 @@ Installing external packages ---------------------------- -On Windows, there is no standard place where to download, build and -install third-party libraries. We recommend installing them in the parent -directory of the pypy checkout. For example, if you installed pypy in -``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You must then set the -INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. +We uses a `repository` parallel to pypy to hold binary compiled versions of the +build dependencies for windows. As part of the `rpython` setup stage, environment +variables will be set to use these dependencies. The repository has a README +file on how to replicate, and a branch for each supported platform. You may run + the `get_externals.py` utility to checkout the proper branch for your platform +and PyPy version. - -Abridged method (using Visual Studio 2008) ------------------------------------------- - -Download the versions of all the external packages from -https://bitbucket.org/pypy/pypy/downloads/local_59.zip -(for post-5.8 builds) with sha256 checksum -``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` -https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(to reproduce 5.8 builds) with sha256 checksum -``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or -https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip -(for 2.4 release and later) or -https://bitbucket.org/pypy/pypy/downloads/local.zip -(for pre-2.4 versions) -Then expand it into the base directory (base_dir) and modify your environment -to reflect this:: - - set PATH=<base_dir>\bin;%PATH% - set INCLUDE=<base_dir>\include;%INCLUDE% - set LIB=<base_dir>\lib;%LIB% - -Now you should be good to go. If you choose this method, you do not need -to download/build anything else. - -Nonabridged method (building from scratch) ------------------------------------------- - -If you want to, you can rebuild everything from scratch by continuing. - - -The Boehm garbage collector -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This library is needed if you plan to use the ``--gc=boehm`` translation -option (this is the default at some optimization levels like ``-O1``, -but unneeded for high-performance translations like ``-O2``). -You may get it at -http://hboehm.info/gc/gc_source/gc-7.1.tar.gz - -Versions 7.0 and 7.1 are known to work; the 6.x series won't work with -RPython. Unpack this folder in the base directory. -The default GC_abort(...) function in misc.c will try to open a MessageBox. -You may want to disable this with the following patch:: - - --- a/misc.c Sun Apr 20 14:08:27 2014 +0300 - +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300 - @@ -1058,7 +1058,7 @@ - #ifndef PCR - void GC_abort(const char *msg) - { - -# if defined(MSWIN32) - +# if 0 && defined(MSWIN32) - (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK); - # else - GC_err_printf("%s\n", msg); - -Then open a command prompt:: - - cd gc-7.1 - nmake -f NT_THREADS_MAKEFILE - copy Release\gc.dll <somewhere in the PATH> - - -The zlib compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in -the base directory. Then compile:: - - cd zlib-1.2.11 - nmake -f win32\Makefile.msc - copy zlib.lib <somewhere in LIB> - copy zlib.h zconf.h <somewhere in INCLUDE> - copy zlib1.dll <in PATH> # (needed for tests via ll2ctypes) - - -The bz2 compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the same version of bz2 used by python and compile as a static library:: - - svn export http://svn.python.org/projects/external/bzip2-1.0.6 - cd bzip2-1.0.6 - nmake -f makefile.msc - copy libbz2.lib <somewhere in LIB> - copy bzlib.h <somewhere in INCLUDE> - - -The sqlite3 database library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi -wrapper is compiled when the module is imported for the first time. -The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility. - - -The expat XML parser -~~~~~~~~~~~~~~~~~~~~ - -CPython compiles expat from source as part of the build. PyPy uses the same -code base, but expects to link to a static lib of expat. Here are instructions -to reproduce the static lib in version 2.2.4. - -Download the source code of expat: https://github.com/libexpat/libexpat. -``git checkout`` the proper tag, in this case ``R_2_2_4``. Run -``vcvars.bat`` to set up the visual compiler tools, and CD into the source -directory. Create a file ``stdbool.h`` with the content - -.. code-block:: c - - #pragma once - - #define false 0 - #define true 1 - - #define bool int - -and put it in a place on the ``INCLUDE`` path, or create it in the local -directory and add ``.`` to the ``INCLUDE`` path:: - - SET INCLUDE=%INCLUDE%;. - -Then compile all the ``*.c`` file into ``*.obj``:: - - cl.exe /nologo /MD /O2 *c /c - rem for debug - cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c - -You may need to move some variable declarations to the beginning of the -function, to be compliant with C89 standard. Here is the diff for version 2.2.4 - -.. code-block:: diff - - diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c - index 007aed0..a2dcaad 100644 - --- a/expat/lib/xmltok.c - +++ b/expat/lib/xmltok.c - @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), - /* Avoid copying partial characters (due to limited space). */ - const ptrdiff_t bytesAvailable = fromLim - *fromP; - const ptrdiff_t bytesStorable = toLim - *toP; - + const char * fromLimBefore; - + ptrdiff_t bytesToCopy; - if (bytesAvailable > bytesStorable) { - fromLim = *fromP + bytesStorable; - output_exhausted = true; - } - - /* Avoid copying partial characters (from incomplete input). */ - - const char * const fromLimBefore = fromLim; - + fromLimBefore = fromLim; - align_limit_to_full_utf8_characters(*fromP, &fromLim); - if (fromLim < fromLimBefore) { - input_incomplete = true; - } - - - const ptrdiff_t bytesToCopy = fromLim - *fromP; - + bytesToCopy = fromLim - *fromP; - memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); - *fromP += bytesToCopy; - *toP += bytesToCopy; - - -Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: - - cl /LD *.obj libexpat.def /Felibexpat.dll - rem for debug - rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll - - rem this will override the export library created in the step above - rem but tests do not need the export library, they load the dll dynamically - lib *.obj /out:libexpat.lib - -Then, copy - -- ``libexpat.lib`` into LIB -- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE -- ``libexpat.dll`` into PATH - - -The OpenSSL library -~~~~~~~~~~~~~~~~~~~ - -OpenSSL needs a Perl interpreter to configure its makefile. You may -use the one distributed by ActiveState, or the one from cygwin.:: - - svn export http://svn.python.org/projects/external/openssl-1.0.2k - cd openssl-1.0.2k - perl Configure VC-WIN32 no-idea no-mdc2 - ms\do_ms.bat - nmake -f ms\nt.mak install - copy out32\*.lib <somewhere in LIB> - xcopy /S include\openssl <somewhere in INCLUDE> - -For tests you will also need the dlls:: - nmake -f ms\ntdll.mak install - copy out32dll\*.dll <somewhere in PATH> - -TkInter module support -~~~~~~~~~~~~~~~~~~~~~~ - -Note that much of this is taken from the cpython build process. -Tkinter is imported via cffi, so the module is optional. To recreate the tcltk -directory found for the release script, create the dlls, libs, headers and -runtime by running:: - - svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 - svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 - cd tcl85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all - nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install - cd ..\..\tk85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install - copy ..\..\tcltk\bin\* <somewhere in PATH> - copy ..\..\tcltk\lib\*.lib <somewhere in LIB> - xcopy /S ..\..\tcltk\include <somewhere in INCLUDE> - -The lzma compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Python 3.3 ship with CFFI wrappers for the lzma library, which can be -downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version -5.0.5, a prebuilt version can be downloaded from -http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature -http://tukaani.org/xz/xz-5.0.5-windows.zip.sig - -Then copy the headers to the include directory, rename ``liblzma.a`` to -``lzma.lib`` and copy it to the lib directory - +.. _repository: https://bitbucket.org/pypy/external Using the mingw compiler ------------------------ diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -87,7 +87,6 @@ 'get_hidden_tb' : 'interp_magic.get_hidden_tb', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', - 'validate_fd' : 'interp_magic.validate_fd', 'resizelist_hint' : 'interp_magic.resizelist_hint', 'newlist_hint' : 'interp_magic.newlist_hint', 'add_memory_pressure' : 'interp_magic.add_memory_pressure', diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -107,14 +107,6 @@ raise oefmt(space.w_TypeError, "expecting dict or list or set object") return space.newtext(name) - -@unwrap_spec(fd='c_int') -def validate_fd(space, fd): - try: - rposix.validate_fd(fd) - except OSError as e: - raise wrap_oserror(space, e) - def get_console_cp(space): from rpython.rlib import rwin32 # Windows only return space.newtuple([ diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -30,7 +30,7 @@ from pypy.module.__builtin__.interp_classobj import W_ClassObject from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel -from rpython.rlib.rposix import is_valid_fd, validate_fd +from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize from pypy.module import exceptions @@ -96,25 +96,24 @@ dash = '' def fclose(fp): - if not is_valid_fd(c_fileno(fp)): + try: + with FdValidator(c_fileno(fp)): + return c_fclose(fp) + except IOError: return -1 - return c_fclose(fp) def fwrite(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fwrite(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fwrite(buf, sz, n, fp) def fread(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fread(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fread(buf, sz, n, fp) _feof = rffi.llexternal('feof', [FILEP], rffi.INT) def feof(fp): - validate_fd(c_fileno(fp)) - return _feof(fp) - -def is_valid_fp(fp): - return is_valid_fd(c_fileno(fp)) + with FdValidator(c_fileno(fp)): + return _feof(fp) pypy_decl = 'pypy_decl.h' udir.join(pypy_decl).write("/* Will be filled later */\n") diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -5,7 +5,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, - cpython_struct, is_valid_fp) + cpython_struct) from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject @@ -155,22 +155,19 @@ BUF_SIZE = 8192 source = "" filename = rffi.charp2str(filename) - buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') - if not is_valid_fp(fp): - lltype.free(buf, flavor='raw') - PyErr_SetFromErrno(space, space.w_IOError) - return None - try: + with rffi.scoped_alloc_buffer(BUF_SIZE) as buf: while True: - count = fread(buf, 1, BUF_SIZE, fp) + try: + count = fread(buf.raw, 1, BUF_SIZE, fp) + except OSError: + PyErr_SetFromErrno(space, space.w_IOError) + return count = rffi.cast(lltype.Signed, count) - source += rffi.charpsize2str(buf, count) + source += rffi.charpsize2str(buf.raw, count) if count < BUF_SIZE: if feof(fp): break PyErr_SetFromErrno(space, space.w_IOError) - finally: - lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) # Undocumented function! diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -13,7 +13,7 @@ PyEval_GetBuiltins, PyEval_GetLocals, PyEval_GetGlobals, _PyEval_SliceIndex) from pypy.module.cpyext.api import ( - c_fopen, c_fclose, c_fileno, Py_ssize_tP, is_valid_fd) + c_fopen, c_fclose, c_fileno, Py_ssize_tP) from pypy.module.cpyext.pyobject import get_w_obj_and_decref from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError @@ -150,7 +150,6 @@ os.close(c_fileno(fp)) with raises_w(space, IOError): PyRun_File(space, fp, filename, Py_file_input, w_globals, w_locals) - if is_valid_fd(c_fileno(fp)): c_fclose(fp) rffi.free_charp(filename) diff --git a/pypy/module/faulthandler/handler.py b/pypy/module/faulthandler/handler.py --- a/pypy/module/faulthandler/handler.py +++ b/pypy/module/faulthandler/handler.py @@ -1,6 +1,5 @@ import os from rpython.rtyper.lltypesystem import lltype, llmemory, rffi -from rpython.rlib.rposix import is_valid_fd from rpython.rlib.rarithmetic import widen, ovfcheck_float_to_longlong from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.annlowlevel import llhelper @@ -35,7 +34,7 @@ raise oefmt(space.w_RuntimeError, "sys.stderr is None") elif space.isinstance_w(w_file, space.w_int): fd = space.c_int_w(w_file) - if fd < 0 or not is_valid_fd(fd): + if fd < 0: raise oefmt(space.w_ValueError, "file is not a valid file descriptor") return fd, None diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -1,7 +1,6 @@ # NOT_RPYTHON from _structseq import structseqtype, structseqfield -from __pypy__ import validate_fd # XXX we need a way to access the current module's globals more directly... import sys @@ -80,25 +79,27 @@ f_flag = structseqfield(8) f_namemax = structseqfield(9) +# Capture file.fdopen at import time, as some code replaces +# __builtins__.file with a custom function. +_fdopen = file.fdopen + if osname == 'posix': # POSIX: we want to check the file descriptor when fdopen() is called, # not later when we read or write data. So we call fstat(), letting # it raise if fd is invalid. - _validate_fd = posix.fstat + def fdopen(fd, mode='r', buffering=-1): + """fdopen(fd [, mode='r' [, buffering]]) -> file_object + + Return an open file object connected to a file descriptor.""" + posix.fstat(fd) + return _fdopen(fd, mode, buffering) + else: - _validate_fd = validate_fd + def fdopen(fd, mode='r', buffering=-1): + """fdopen(fd [, mode='r' [, buffering]]) -> file_object -# Capture file.fdopen at import time, as some code replaces -# __builtins__.file with a custom function. -_fdopen = file.fdopen - -def fdopen(fd, mode='r', buffering=-1): - """fdopen(fd [, mode='r' [, buffering]]) -> file_object - - Return an open file object connected to a file descriptor.""" - _validate_fd(fd) - return _fdopen(fd, mode, buffering) - + Return an open file object connected to a file descriptor.""" + return _fdopen(fd, mode, buffering) def tmpfile(): """Create a temporary file. diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -18,7 +18,7 @@ USEMODULES += ['fcntl'] else: # On windows, os.popen uses the subprocess module - USEMODULES += ['_rawffi', 'thread', 'signal'] + USEMODULES += ['_rawffi', 'thread', 'signal', '_cffi_backend'] def setup_module(mod): mod.space = gettestobjspace(usemodules=USEMODULES) @@ -300,8 +300,13 @@ # There used to be code here to ensure that fcntl is not faked # but we can't do that cleanly any more - exc = raises(OSError, posix.fdopen, fd) - assert exc.value.errno == errno.EBADF + try: + fid = posix.fdopen(fd) + fid.read(10) + except IOError as e: + assert e.errno == errno.EBADF + else: + assert False, "using result of fdopen(fd) on closed file must raise" def test_fdopen_hackedbuiltins(self): "Same test, with __builtins__.file removed" @@ -331,8 +336,17 @@ path = self.path posix = self.posix fd = posix.open(path, posix.O_RDONLY) - exc = raises(OSError, posix.fdopen, fd, 'w') - assert str(exc.value) == "[Errno 22] Invalid argument" + # compatability issue - using Visual Studio 10 and above no + # longer raises on fid creation, only when _using_ fid + # win32 python2 raises IOError on flush(), win32 python3 raises OSError + try: + fid = posix.fdopen(fd, 'w') + fid.write('abc') + fid.flush() + except (OSError, IOError) as e: + assert e.errno in (9, 22) + else: + assert False, "expected OSError" posix.close(fd) # fd should not be closed def test_getcwd(self): diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -151,9 +151,10 @@ if (rffi.TIME_T in args or rffi.TIME_TP in args or result in (rffi.TIME_T, rffi.TIME_TP)): name = '_' + name + '64' + _calling_conv = kwds.pop('calling_conv', calling_conv) return rffi.llexternal(name, args, result, compilation_info=eci, - calling_conv=calling_conv, + calling_conv=_calling_conv, releasegil=False, **kwds) @@ -187,20 +188,34 @@ "RPY_EXTERN " "int pypy_get_daylight();\n" "RPY_EXTERN " - "char** pypy_get_tzname();\n" + "int pypy_get_tzname(size_t, int, char*);\n" "RPY_EXTERN " "void pypy__tzset();"], separate_module_sources = [""" - long pypy_get_timezone() { return timezone; } - int pypy_get_daylight() { return daylight; } - char** pypy_get_tzname() { return tzname; } - void pypy__tzset() { _tzset(); } + long pypy_get_timezone() { + long timezone; + _get_timezone(&timezone); + return timezone; + }; + int pypy_get_daylight() { + int daylight; + _get_daylight(&daylight); + return daylight; + }; + int pypy_get_tzname(size_t len, int index, char * tzname) { + size_t s; + errno_t ret = _get_tzname(&s, tzname, len, index); + return (int)s; + }; + void pypy__tzset() { _tzset(); } """]) # Ensure sure that we use _tzset() and timezone from the same C Runtime. c_tzset = external('pypy__tzset', [], lltype.Void, win_eci) c_get_timezone = external('pypy_get_timezone', [], rffi.LONG, win_eci) c_get_daylight = external('pypy_get_daylight', [], rffi.INT, win_eci) - c_get_tzname = external('pypy_get_tzname', [], rffi.CCHARPP, win_eci) + c_get_tzname = external('pypy_get_tzname', + [rffi.SIZE_T, rffi.INT, rffi.CCHARP], + rffi.INT, win_eci, calling_conv='c') c_strftime = external('strftime', [rffi.CCHARP, rffi.SIZE_T, rffi.CCHARP, TM_P], rffi.SIZE_T) @@ -221,8 +236,11 @@ timezone = c_get_timezone() altzone = timezone - 3600 daylight = c_get_daylight() - tzname_ptr = c_get_tzname() - tzname = rffi.charp2str(tzname_ptr[0]), rffi.charp2str(tzname_ptr[1]) + with rffi.scoped_alloc_buffer(100) as buf: + s = c_get_tzname(100, 0, buf.raw) + tzname[0] = buf.str(s) + s = c_get_tzname(100, 1, buf.raw) + tzname[1] = buf.str(s) if _POSIX: if _CYGWIN: diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -296,7 +296,8 @@ def get_libc_name(): return rwin32.GetModuleFileName(get_libc_handle()) - assert "msvcr" in get_libc_name().lower(), \ + libc_name = get_libc_name().lower() + assert "msvcr" in libc_name or 'ucrtbase' in libc_name, \ "Suspect msvcrt library: %s" % (get_libc_name(),) elif _MINGW: def get_libc_name(): diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -173,10 +173,10 @@ def create_fdopen_rfile(fd, mode="r", buffering=-1): newmode = _sanitize_mode(mode) - rposix.validate_fd(fd) ll_mode = rffi.str2charp(newmode) try: - ll_file = c_fdopen(fd, ll_mode) + with rposix.FdValidator(fd): + ll_file = c_fdopen(fd, ll_mode) if not ll_file: errno = rposix.get_saved_errno() raise OSError(errno, os.strerror(errno)) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -39,11 +39,12 @@ if os.name == 'nt': if platform.name == 'msvc': - includes=['errno.h','stdio.h'] + includes=['errno.h','stdio.h', 'stdlib.h'] else: includes=['errno.h','stdio.h', 'stdint.h'] separate_module_sources =[''' - /* Lifted completely from CPython 3.3 Modules/posix_module.c */ + /* Lifted completely from CPython 3 Modules/posixmodule.c */ + #if defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900 #include <malloc.h> /* for _msize */ typedef struct { intptr_t osfhnd; @@ -95,6 +96,46 @@ errno = EBADF; return 0; } + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; + #elif defined _MSC_VER + RPY_EXTERN int _PyVerify_fd(int fd) + { + return 1; + } + static void __cdecl _Py_silent_invalid_parameter_handler( + wchar_t const* expression, + wchar_t const* function, + wchar_t const* file, + unsigned int line, + uintptr_t pReserved) { + wprintf(L"Invalid parameter detected in function %s." + L" File: %s Line: %d\\n", function, file, line); + wprintf(L"Expression: %s\\n", expression); + } + + RPY_EXTERN void* enter_suppress_iph(void) + { + void* ret = _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler); + /*fprintf(stdout, "setting %p returning %p\\n", (void*)_Py_silent_invalid_parameter_handler, ret);*/ + return ret; + } + RPY_EXTERN void exit_suppress_iph(void* old_handler) + { + void * ret; + _invalid_parameter_handler _handler = (_invalid_parameter_handler)old_handler; + ret = _set_thread_local_invalid_parameter_handler(_handler); + /*fprintf(stdout, "exiting, setting %p returning %p\\n", old_handler, ret);*/ + } + + #else + RPY_EXTERN int _PyVerify_fd(int fd) + { + return 1; + } + RPY_EXTERN void* enter_suppress_iph(void) {return (void*)NULL;}; + RPY_EXTERN void exit_suppress_iph(void* handle) {}; + #endif ''',] post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] else: @@ -193,50 +234,6 @@ else: rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below - - -if os.name == 'nt': - is_valid_fd = jit.dont_look_inside(rffi.llexternal( - "_PyVerify_fd", [rffi.INT], rffi.INT, - compilation_info=errno_eci, - )) - @enforceargs(int) - def validate_fd(fd): - if not is_valid_fd(fd): - from errno import EBADF - raise OSError(EBADF, 'Bad file descriptor') - - def _bound_for_write(fd, count): - if count > 32767 and c_isatty(fd): - # CPython Issue #11395, PyPy Issue #2636: the Windows console - # returns an error (12: not enough space error) on writing into - # stdout if stdout mode is binary and the length is greater than - # 66,000 bytes (or less, depending on heap usage). Can't easily - # test that, because we need 'fd' to be non-redirected... - count = 32767 - elif count > 0x7fffffff: - count = 0x7fffffff - return count -else: - def is_valid_fd(fd): - return 1 - - @enforceargs(int) - def validate_fd(fd): - pass - - def _bound_for_write(fd, count): - return count - -def closerange(fd_low, fd_high): - # this behaves like os.closerange() from Python 2.6. - for fd in xrange(fd_low, fd_high): - try: - if is_valid_fd(fd): - os.close(fd) - except OSError: - pass - if _WIN32: includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] libraries = [] @@ -258,11 +255,80 @@ if sys.platform.startswith('freebsd') or sys.platform.startswith('openbsd'): includes.append('sys/ttycom.h') libraries = ['util'] + eci = ExternalCompilationInfo( includes=includes, libraries=libraries, ) +def external(name, args, result, compilation_info=eci, **kwds): + return rffi.llexternal(name, args, result, + compilation_info=compilation_info, **kwds) + + +if os.name == 'nt': + is_valid_fd = jit.dont_look_inside(external("_PyVerify_fd", [rffi.INT], + rffi.INT, compilation_info=errno_eci, + )) + c_enter_suppress_iph = jit.dont_look_inside(external("enter_suppress_iph", + [], rffi.VOIDP, compilation_info=errno_eci)) + c_exit_suppress_iph = jit.dont_look_inside(external("exit_suppress_iph", + [rffi.VOIDP], lltype.Void, + compilation_info=errno_eci)) + + @enforceargs(int) + def _validate_fd(fd): + if not is_valid_fd(fd): + from errno import EBADF + raise OSError(EBADF, 'Bad file descriptor') + + class FdValidator(object): + + def __init__(self, fd): + _validate_fd(fd) + + def __enter__(self): + self.invalid_param_hndlr = c_enter_suppress_iph() + return self + + def __exit__(self, *args): + c_exit_suppress_iph(self.invalid_param_hndlr) + + def _bound_for_write(fd, count): + if count > 32767 and c_isatty(fd): + # CPython Issue #11395, PyPy Issue #2636: the Windows console + # returns an error (12: not enough space error) on writing into + # stdout if stdout mode is binary and the length is greater than + # 66,000 bytes (or less, depending on heap usage). Can't easily + # test that, because we need 'fd' to be non-redirected... + count = 32767 + elif count > 0x7fffffff: + count = 0x7fffffff + return count +else: + class FdValidator(object): + + def __init__(self, fd): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def _bound_for_write(fd, count): + return count + +def closerange(fd_low, fd_high): + # this behaves like os.closerange() from Python 2.6. + for fd in xrange(fd_low, fd_high): + try: + with FdValidator(fd): + os.close(fd) + except OSError: + pass + class CConfig: _compilation_info_ = eci SEEK_SET = rffi_platform.DefinedConstantInteger('SEEK_SET') @@ -315,10 +381,6 @@ config = rffi_platform.configure(CConfig) globals().update(config) -def external(name, args, result, compilation_info=eci, **kwds): - return rffi.llexternal(name, args, result, - compilation_info=compilation_info, **kwds) - # For now we require off_t to be the same size as LONGLONG, which is the # interface required by callers of functions that thake an argument of type # off_t. @@ -410,12 +472,12 @@ return result def _dup(fd, inheritable=True): - validate_fd(fd) - if inheritable: - res = c_dup(fd) - else: - res = c_dup_noninheritable(fd) - return res + with FdValidator(fd): + if inheritable: + res = c_dup(fd) + else: + res = c_dup_noninheritable(fd) + return res @replace_os_function('dup') def dup(fd, inheritable=True): @@ -424,12 +486,12 @@ @replace_os_function('dup2') def dup2(fd, newfd, inheritable=True): - validate_fd(fd) - if inheritable: - res = c_dup2(fd, newfd) - else: - res = c_dup2_noninheritable(fd, newfd) - handle_posix_error('dup2', res) + with FdValidator(fd): + if inheritable: + res = c_dup2(fd, newfd) + else: + res = c_dup2_noninheritable(fd, newfd) + handle_posix_error('dup2', res) #___________________________________________________________________ @@ -457,25 +519,26 @@ def read(fd, count): if count < 0: raise OSError(errno.EINVAL, None) - validate_fd(fd) - with rffi.scoped_alloc_buffer(count) as buf: - void_buf = rffi.cast(rffi.VOIDP, buf.raw) - got = handle_posix_error('read', c_read(fd, void_buf, count)) - return buf.str(got) + with FdValidator(fd): + with rffi.scoped_alloc_buffer(count) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + got = handle_posix_error('read', c_read(fd, void_buf, count)) + return buf.str(got) @replace_os_function('write') @enforceargs(int, None) def write(fd, data): count = len(data) - validate_fd(fd) - count = _bound_for_write(fd, count) - with rffi.scoped_nonmovingbuffer(data) as buf: - return handle_posix_error('write', c_write(fd, buf, count)) + with FdValidator(fd): + count = _bound_for_write(fd, count) + with rffi.scoped_nonmovingbuffer(data) as buf: + ret = c_write(fd, buf, count) + return handle_posix_error('write', ret) @replace_os_function('close') def close(fd): - validate_fd(fd) - handle_posix_error('close', c_close(fd)) + with FdValidator(fd): + handle_posix_error('close', c_close(fd)) c_lseek = external('_lseeki64' if _WIN32 else 'lseek', [rffi.INT, rffi.LONGLONG, rffi.INT], rffi.LONGLONG, @@ -483,15 +546,15 @@ @replace_os_function('lseek') def lseek(fd, pos, how): - validate_fd(fd) - if SEEK_SET is not None: - if how == 0: - how = SEEK_SET - elif how == 1: - how = SEEK_CUR - elif how == 2: - how = SEEK_END - return handle_posix_error('lseek', c_lseek(fd, pos, how)) + with FdValidator(fd): + if SEEK_SET is not None: + if how == 0: + how = SEEK_SET + elif how == 1: + how = SEEK_CUR + elif how == 2: + how = SEEK_END + return handle_posix_error('lseek', c_lseek(fd, pos, how)) if not _WIN32: c_pread = external('pread', @@ -505,7 +568,6 @@ def pread(fd, count, offset): if count < 0: raise OSError(errno.EINVAL, None) - validate_fd(fd) with rffi.scoped_alloc_buffer(count) as buf: void_buf = rffi.cast(rffi.VOIDP, buf.raw) return buf.str(handle_posix_error('pread', c_pread(fd, void_buf, count, offset))) @@ -513,7 +575,6 @@ @enforceargs(int, None, None) def pwrite(fd, data, offset): count = len(data) - validate_fd(fd) with rffi.scoped_nonmovingbuffer(data) as buf: return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset)) @@ -524,7 +585,6 @@ @enforceargs(int, None, None) def posix_fallocate(fd, offset, length): - validate_fd(fd) return handle_posix_error('posix_fallocate', c_posix_fallocate(fd, offset, length)) if HAVE_FADVISE: @@ -546,7 +606,6 @@ @enforceargs(int, None, None, int) def posix_fadvise(fd, offset, length, advice): - validate_fd(fd) error = c_posix_fadvise(fd, offset, length, advice) error = widen(error) if error != 0: @@ -557,7 +616,6 @@ save_err=rffi.RFFI_SAVE_ERRNO) @enforceargs(int, None, None) def lockf(fd, cmd, length): - validate_fd(fd) return handle_posix_error('lockf', c_lockf(fd, cmd, length)) c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT, @@ -571,18 +629,18 @@ @replace_os_function('ftruncate') def ftruncate(fd, length): - validate_fd(fd) - handle_posix_error('ftruncate', c_ftruncate(fd, length)) + with FdValidator(fd): + handle_posix_error('ftruncate', c_ftruncate(fd, length)) @replace_os_function('fsync') def fsync(fd): - validate_fd(fd) - handle_posix_error('fsync', c_fsync(fd)) + with FdValidator(fd): + handle_posix_error('fsync', c_fsync(fd)) @replace_os_function('fdatasync') def fdatasync(fd): - validate_fd(fd) - handle_posix_error('fdatasync', c_fdatasync(fd)) + with FdValidator(fd): + handle_posix_error('fdatasync', c_fdatasync(fd)) def sync(): c_sync() @@ -649,8 +707,8 @@ @replace_os_function('fchdir') def fchdir(fd): - validate_fd(fd) - handle_posix_error('fchdir', c_fchdir(fd)) + with FdValidator(fd): + handle_posix_error('fchdir', c_fchdir(fd)) @replace_os_function('access') @specialize.argtype(0) @@ -1090,9 +1148,9 @@ @replace_os_function('isatty') def isatty(fd): - if not is_valid_fd(fd): - return False - return c_isatty(fd) != 0 + with FdValidator(fd): + return c_isatty(fd) != 0 + return False c_ttyname = external('ttyname', [lltype.Signed], rffi.CCHARP, releasegil=False, diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -117,7 +117,7 @@ from rpython.translator.platform import host_factory static_platform = host_factory() if static_platform.name == 'msvc': - defines += ' PROCESS_QUERY_LIMITED_INFORMATION' + defines += ' PROCESS_QUERY_LIMITED_INFORMATION' for name in defines.split(): locals()[name] = rffi_platform.ConstantInteger(name) @@ -171,7 +171,7 @@ function, if that C function was declared with the flag llexternal(..., save_err=RFFI_SAVE_LASTERROR | RFFI_ALT_ERRNO). Functions without that flag don't change the saved LastError. - Alternatively, if the function was declared + Alternatively, if the function was declared RFFI_SAVE_WSALASTERROR | RFFI_ALT_ERRNO, then the value of the C-level WSAGetLastError() is saved instead (into the same "saved alt LastError" variable). @@ -218,9 +218,9 @@ _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) def get_osfhandle(fd): - from rpython.rlib.rposix import validate_fd - validate_fd(fd) - handle = _get_osfhandle(fd) + from rpython.rlib.rposix import FdValidator + with FdValidator(fd): + handle = _get_osfhandle(fd) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle @@ -231,45 +231,10 @@ in the dict.""" # Prior to Visual Studio 8, the MSVCRT dll doesn't export the # _dosmaperr() function, which is available only when compiled - # against the static CRT library. - from rpython.translator.platform import host_factory - static_platform = host_factory() - if static_platform.name == 'msvc': - static_platform.cflags = ['/MT'] # static CRT - static_platform.version = 0 # no manifest - cfile = udir.join('dosmaperr.c') - cfile.write(r''' - #include <errno.h> - #include <WinError.h> - #include <stdio.h> - #ifdef __GNUC__ - #define _dosmaperr mingw_dosmaperr - #endif - int main() - { - int i; - for(i=1; i < 65000; i++) { - _dosmaperr(i); - if (errno == EINVAL) { - /* CPython issue #12802 */ - if (i == ERROR_DIRECTORY) - errno = ENOTDIR; - else - continue; - } - printf("%d\t%d\n", i, errno); - } - return 0; - }''') - try: - exename = static_platform.compile( - [cfile], ExternalCompilationInfo(), - outputfilename = "dosmaperr", - standalone=True) - except (CompilationError, WindowsError): - # Fallback for the mingw32 compiler - assert static_platform.name == 'mingw32' - errors = { + # against the static CRT library. After Visual Studio 9, this + # private function seems to be gone, so use a static map, from + # CPython PC/errmap.h + errors = { 2: 2, 3: 2, 4: 24, 5: 13, 6: 9, 7: 12, 8: 12, 9: 12, 10: 7, 11: 8, 15: 2, 16: 13, 17: 18, 18: 2, 19: 13, 20: 13, 21: 13, 22: 13, 23: 13, 24: 13, 25: 13, 26: 13, 27: 13, 28: 13, @@ -279,12 +244,8 @@ 132: 13, 145: 41, 158: 13, 161: 2, 164: 11, 167: 13, 183: 17, 188: 8, 189: 8, 190: 8, 191: 8, 192: 8, 193: 8, 194: 8, 195: 8, 196: 8, 197: 8, 198: 8, 199: 8, 200: 8, 201: 8, - 202: 8, 206: 2, 215: 11, 267: 20, 1816: 12, + 202: 8, 206: 2, 215: 11, 232: 32, 267: 20, 1816: 12, } - else: - output = os.popen(str(exename)) - errors = dict(map(int, line.split()) - for line in output) return errors, errno.EINVAL # A bit like strerror... @@ -299,7 +260,7 @@ buf[0] = lltype.nullptr(rffi.CCHARP.TO) try: msglen = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, None, rffi.cast(DWORD, code), @@ -330,7 +291,7 @@ buf[0] = lltype.nullptr(rffi.CWCHARP.TO) try: msglen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, None, rffi.cast(DWORD, code), diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -15,19 +15,6 @@ py.test.raises(OSError, rwin32.get_osfhandle, fd) rwin32.get_osfhandle(0) -def test_get_osfhandle_raising(): - #try to test what kind of exception get_osfhandle raises w/out fd validation - py.test.skip('Crashes python') - fid = open(str(udir.join('validate_test.txt')), 'w') - fd = fid.fileno() - fid.close() - def validate_fd(fd): - return 1 - _validate_fd = rwin32.validate_fd - rwin32.validate_fd = validate_fd - raises(WindowsError, rwin32.get_osfhandle, fd) - rwin32.validate_fd = _validate_fd - def test_open_process(): pid = rwin32.GetCurrentProcessId() assert pid != 0 diff --git a/rpython/tool/setuptools_msvc.py b/rpython/tool/setuptools_msvc.py new file mode 100644 --- /dev/null +++ b/rpython/tool/setuptools_msvc.py @@ -0,0 +1,1308 @@ +""" +Improved support for Microsoft Visual C++ compilers. + +Known supported compilers: +-------------------------- +Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + Microsoft Windows SDK 6.1 (x86, x64, ia64) + Microsoft Windows SDK 7.0 (x86, x64, ia64) + +Microsoft Visual C++ 10.0: + Microsoft Windows SDK 7.1 (x86, x64, ia64) + +Microsoft Visual C++ 14.0: + Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) + Microsoft Visual Studio 2017 (x86, x64, arm, arm64) + Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64) +""" + +# copied as-is from python3 +# modified to: +# - commented out monkey patching +# - lookup() returns str not unicode + +import os +import sys +import platform +import itertools +import distutils.errors +from pkg_resources.extern.packaging.version import LegacyVersion + +from setuptools.extern.six.moves import filterfalse + +#from .monkey import get_unpatched + +if platform.system() == 'Windows': + from setuptools.extern.six.moves import winreg + safe_env = os.environ +else: + """ + Mock winreg and environ so the module can be imported + on this platform. + """ + + class winreg: + HKEY_USERS = None + HKEY_CURRENT_USER = None + HKEY_LOCAL_MACHINE = None + HKEY_CLASSES_ROOT = None + + safe_env = dict() + +_msvc9_suppress_errors = ( + # msvc9compiler isn't available on some platforms + ImportError, + + # msvc9compiler raises DistutilsPlatformError in some + # environments. See #1118. + distutils.errors.DistutilsPlatformError, +) + +try: + from distutils.msvc9compiler import Reg +except _msvc9_suppress_errors: + pass + + +def msvc9_find_vcvarsall(version): + """ + Patched "distutils.msvc9compiler.find_vcvarsall" to use the standalone + compiler build for Python (VCForPython). Fall back to original behavior + when the standalone compiler is not available. + + Redirect the path of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + + Parameters + ---------- + version: float + Required Microsoft Visual C++ version. + + Return + ------ + vcvarsall.bat path: str + """ + VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f' + key = VC_BASE % ('', version) + try: + # Per-user installs register the compiler path here + productdir = Reg.get_value(key, "installdir") + except KeyError: + try: + # All-user installs on a 64-bit system register here + key = VC_BASE % ('Wow6432Node\\', version) + productdir = Reg.get_value(key, "installdir") + except KeyError: + productdir = None + + if productdir: + vcvarsall = os.path.os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + + return get_unpatched(msvc9_find_vcvarsall)(version) + + +def msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs): + """ + Patched "distutils.msvc9compiler.query_vcvarsall" for support extra + compilers. + + Set environment without use of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 9.0: + Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + Microsoft Windows SDK 6.1 (x86, x64, ia64) + Microsoft Windows SDK 7.0 (x86, x64, ia64) + + Microsoft Visual C++ 10.0: + Microsoft Windows SDK 7.1 (x86, x64, ia64) + + Parameters + ---------- + ver: float + Required Microsoft Visual C++ version. + arch: str + Target architecture. + + Return + ------ + environment: dict + """ + # Try to get environement from vcvarsall.bat (Classical way) + #try: + # orig = get_unpatched(msvc9_query_vcvarsall) + # return orig(ver, arch, *args, **kwargs) + #except distutils.errors.DistutilsPlatformError: + # # Pass error if Vcvarsall.bat is missing + # pass + #except ValueError: + # # Pass error if environment not set after executing vcvarsall.bat + # pass + + # If error, try to set environment directly + try: + return EnvironmentInfo(arch, ver).return_env() + except distutils.errors.DistutilsPlatformError as exc: + _augment_exception(exc, ver, arch) + raise + + +def msvc14_get_vc_env(plat_spec): + """ + Patched "distutils._msvccompiler._get_vc_env" for support extra + compilers. + + Set environment without use of "vcvarsall.bat". + + Known supported compilers + ------------------------- + Microsoft Visual C++ 14.0: + Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) + Microsoft Visual Studio 2017 (x86, x64, arm, arm64) + Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64) + + Parameters + ---------- + plat_spec: str + Target architecture. + + Return + ------ + environment: dict + """ + # Try to get environment from vcvarsall.bat (Classical way) + #try: + # return get_unpatched(msvc14_get_vc_env)(plat_spec) + #except distutils.errors.DistutilsPlatformError: + # Pass error Vcvarsall.bat is missing + # pass + + # If error, try to set environment directly + try: + return EnvironmentInfo(plat_spec, vc_min_ver=14.0).return_env() + except distutils.errors.DistutilsPlatformError as exc: + _augment_exception(exc, 14.0) + raise + + +def msvc14_gen_lib_options(*args, **kwargs): + """ + Patched "distutils._msvccompiler.gen_lib_options" for fix + compatibility between "numpy.distutils" and "distutils._msvccompiler" + (for Numpy < 1.11.2) + """ + if "numpy.distutils" in sys.modules: + import numpy as np + if LegacyVersion(np.__version__) < LegacyVersion('1.11.2'): + return np.distutils.ccompiler.gen_lib_options(*args, **kwargs) + return get_unpatched(msvc14_gen_lib_options)(*args, **kwargs) + + +def _augment_exception(exc, version, arch=''): + """ + Add details to the exception message to help guide the user + as to what action will resolve it. + """ + # Error if MSVC++ directory not found or environment not set + message = exc.args[0] + + if "vcvarsall" in message.lower() or "visual c" in message.lower(): + # Special error message if MSVC++ not installed + tmpl = 'Microsoft Visual C++ {version:0.1f} is required.' + message = tmpl.format(**locals()) + msdownload = 'www.microsoft.com/download/details.aspx?id=%d' + if version == 9.0: + if arch.lower().find('ia64') > -1: + # For VC++ 9.0, if IA64 support is needed, redirect user + # to Windows SDK 7.0 + message += ' Get it with "Microsoft Windows SDK 7.0": ' + message += msdownload % 3138 + else: + # For VC++ 9.0 redirect user to Vc++ for Python 2.7 : + # This redirection link is maintained by Microsoft. + # Contact vspyt...@microsoft.com if it needs updating. + message += ' Get it from http://aka.ms/vcpython27' + elif version == 10.0: + # For VC++ 10.0 Redirect user to Windows SDK 7.1 + message += ' Get it with "Microsoft Windows SDK 7.1": ' + message += msdownload % 8279 + elif version >= 14.0: + # For VC++ 14.0 Redirect user to Visual C++ Build Tools + message += (' Get it with "Microsoft Visual C++ Build Tools": ' + r'http://landinghub.visualstudio.com/' + 'visual-cpp-build-tools') + + exc.args = (message, ) + + +class PlatformInfo: + """ + Current and Target Architectures informations. + + Parameters + ---------- + arch: str + Target architecture. + """ + current_cpu = safe_env.get('processor_architecture', '').lower() + + def __init__(self, arch): + self.arch = arch.lower().replace('x64', 'amd64') + + @property + def target_cpu(self): + return self.arch[self.arch.find('_') + 1:] + + def target_is_x86(self): + return self.target_cpu == 'x86' + + def current_is_x86(self): + return self.current_cpu == 'x86' + + def current_dir(self, hidex86=False, x64=False): + """ + Current platform specific subfolder. + + Parameters + ---------- + hidex86: bool + return '' and not '\x86' if architecture is x86. + x64: bool + return '\x64' and not '\amd64' if architecture is amd64. + + Return + ------ + subfolder: str + '\target', or '' (see hidex86 parameter) + """ + return ( + '' if (self.current_cpu == 'x86' and hidex86) else + r'\x64' if (self.current_cpu == 'amd64' and x64) else + r'\%s' % self.current_cpu + ) + + def target_dir(self, hidex86=False, x64=False): + r""" + Target platform specific subfolder. + + Parameters + ---------- + hidex86: bool + return '' and not '\x86' if architecture is x86. + x64: bool + return '\x64' and not '\amd64' if architecture is amd64. + + Return + ------ + subfolder: str + '\current', or '' (see hidex86 parameter) + """ + return ( + '' if (self.target_cpu == 'x86' and hidex86) else + r'\x64' if (self.target_cpu == 'amd64' and x64) else + r'\%s' % self.target_cpu + ) + + def cross_dir(self, forcex86=False): + r""" + Cross platform specific subfolder. + + Parameters + ---------- + forcex86: bool + Use 'x86' as current architecture even if current acritecture is + not x86. + + Return + ------ + subfolder: str + '' if target architecture is current architecture, + '\current_target' if not. + """ + current = 'x86' if forcex86 else self.current_cpu + return ( + '' if self.target_cpu == current else + self.target_dir().replace('\\', '\\%s_' % current) + ) + + +class RegistryInfo: + """ + Microsoft Visual Studio related registry informations. + + Parameters + ---------- + platform_info: PlatformInfo + "PlatformInfo" instance. + """ + HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + + def __init__(self, platform_info): + self.pi = platform_info + + @property + def visualstudio(self): + """ + Microsoft Visual Studio root registry key. + """ + return 'VisualStudio' + + @property + def sxs(self): + """ + Microsoft Visual Studio SxS registry key. + """ + return os.path.join(self.visualstudio, 'SxS') + + @property + def vc(self): + """ + Microsoft Visual C++ VC7 registry key. + """ + return os.path.join(self.sxs, 'VC7') + + @property + def vs(self): + """ + Microsoft Visual Studio VS7 registry key. + """ + return os.path.join(self.sxs, 'VS7') + + @property + def vc_for_python(self): + """ + Microsoft Visual C++ for Python registry key. + """ + return r'DevDiv\VCForPython' + + @property + def microsoft_sdk(self): + """ + Microsoft SDK registry key. + """ + return 'Microsoft SDKs' + + @property + def windows_sdk(self): + """ + Microsoft Windows/Platform SDK registry key. + """ + return os.path.join(self.microsoft_sdk, 'Windows') + + @property + def netfx_sdk(self): + """ + Microsoft .NET Framework SDK registry key. + """ + return os.path.join(self.microsoft_sdk, 'NETFXSDK') + + @property + def windows_kits_roots(self): + """ + Microsoft Windows Kits Roots registry key. + """ + return r'Windows Kits\Installed Roots' + + def microsoft(self, key, x86=False): + """ + Return key in Microsoft software registry. + + Parameters + ---------- + key: str + Registry key path where look. + x86: str + Force x86 software registry. + + Return + ------ + str: value + """ + node64 = '' if self.pi.current_is_x86() or x86 else 'Wow6432Node' + return os.path.join('Software', node64, 'Microsoft', key) + + def lookup(self, key, name): + """ + Look for values in registry in Microsoft software registry. + + Parameters + ---------- + key: str + Registry key path where look. + name: str + Value name to find. + + Return + ------ + str: value + """ + KEY_READ = winreg.KEY_READ + openkey = winreg.OpenKey + ms = self.microsoft + for hkey in self.HKEYS: + try: + bkey = openkey(hkey, ms(key), 0, KEY_READ) + except (OSError, IOError): + if not self.pi.current_is_x86(): + try: + bkey = openkey(hkey, ms(key, True), 0, KEY_READ) + except (OSError, IOError): + continue + else: + continue + try: + # modified to return str for python2 + return winreg.QueryValueEx(bkey, name)[0].encode('utf8') + except (OSError, IOError): + pass + + +class SystemInfo: + """ + Microsoft Windows and Visual Studio related system inormations. + + Parameters + ---------- + registry_info: RegistryInfo + "RegistryInfo" instance. + vc_ver: float + Required Microsoft Visual C++ version. + """ + + # Variables and properties in this class use originals CamelCase variables + # names from Microsoft source files for more easy comparaison. + WinDir = safe_env.get('WinDir', '') + ProgramFiles = safe_env.get('ProgramFiles', '') + ProgramFilesx86 = safe_env.get('ProgramFiles(x86)', ProgramFiles) + + def __init__(self, registry_info, vc_ver=None): + self.ri = registry_info + self.pi = self.ri.pi + self.vc_ver = vc_ver or self._find_latest_available_vc_ver() + + def _find_latest_available_vc_ver(self): + try: + return self.find_available_vc_vers()[-1] + except IndexError: + err = 'No Microsoft Visual C++ version found' + raise distutils.errors.DistutilsPlatformError(err) + + def find_available_vc_vers(self): + """ + Find all available Microsoft Visual C++ versions. + """ + ms = self.ri.microsoft + vckeys = (self.ri.vc, self.ri.vc_for_python, self.ri.vs) + vc_vers = [] + for hkey in self.ri.HKEYS: + for key in vckeys: + try: + bkey = winreg.OpenKey(hkey, ms(key), 0, winreg.KEY_READ) + except (OSError, IOError): + continue + subkeys, values, _ = winreg.QueryInfoKey(bkey) + for i in range(values): + try: + ver = float(winreg.EnumValue(bkey, i)[0]) + if ver not in vc_vers: + vc_vers.append(ver) + except ValueError: + pass + for i in range(subkeys): + try: + ver = float(winreg.EnumKey(bkey, i)) + if ver not in vc_vers: + vc_vers.append(ver) + except ValueError: + pass + return sorted(vc_vers) + + @property + def VSInstallDir(self): + """ + Microsoft Visual Studio directory. + """ + # Default path + name = 'Microsoft Visual Studio %0.1f' % self.vc_ver + default = os.path.join(self.ProgramFilesx86, name) + + # Try to get path from registry, if fail use default path + return self.ri.lookup(self.ri.vs, '%0.1f' % self.vc_ver) or default + + @property + def VCInstallDir(self): + """ + Microsoft Visual C++ directory. + """ + self.VSInstallDir + + guess_vc = self._guess_vc() or self._guess_vc_legacy() + + # Try to get "VC++ for Python" path from registry as default path + reg_path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) + python_vc = self.ri.lookup(reg_path, 'installdir') + default_vc = os.path.join(python_vc, 'VC') if python_vc else guess_vc + + # Try to get path from registry, if fail use default path + path = self.ri.lookup(self.ri.vc, '%0.1f' % self.vc_ver) or default_vc + + if not os.path.isdir(path): + msg = 'Microsoft Visual C++ directory not found' + raise distutils.errors.DistutilsPlatformError(msg) + + return path + + def _guess_vc(self): + """ + Locate Visual C for 2017 + """ + if self.vc_ver <= 14.0: + return + + default = r'VC\Tools\MSVC' + guess_vc = os.path.join(self.VSInstallDir, default) + # Subdir with VC exact version as name + try: + vc_exact_ver = os.listdir(guess_vc)[-1] + return os.path.join(guess_vc, vc_exact_ver) + except (OSError, IOError, IndexError): + pass + + def _guess_vc_legacy(self): + """ + Locate Visual C for versions prior to 2017 + """ + default = r'Microsoft Visual Studio %0.1f\VC' % self.vc_ver + return os.path.join(self.ProgramFilesx86, default) + + @property + def WindowsSdkVersion(self): + """ + Microsoft Windows SDK versions for specified MSVC++ version. + """ + if self.vc_ver <= 9.0: + return ('7.0', '6.1', '6.0a') + elif self.vc_ver == 10.0: + return ('7.1', '7.0a') + elif self.vc_ver == 11.0: + return ('8.0', '8.0a') + elif self.vc_ver == 12.0: + return ('8.1', '8.1a') + elif self.vc_ver >= 14.0: + return ('10.0', '8.1') + + @property + def WindowsSdkLastVersion(self): + """ + Microsoft Windows SDK last version + """ + return self._use_last_dir_name(os.path.join( + self.WindowsSdkDir, 'lib')) + + @property + def WindowsSdkDir(self): + """ + Microsoft Windows SDK directory. + """ + sdkdir = '' + for ver in self.WindowsSdkVersion: + # Try to get it from registry + loc = os.path.join(self.ri.windows_sdk, 'v%s' % ver) + sdkdir = self.ri.lookup(loc, 'installationfolder') + if sdkdir: + break + if not sdkdir or not os.path.isdir(sdkdir): + # Try to get "VC++ for Python" version from registry + path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) + install_base = self.ri.lookup(path, 'installdir') + if install_base: + sdkdir = os.path.join(install_base, 'WinSDK') + if not sdkdir or not os.path.isdir(sdkdir): + # If fail, use default new path + for ver in self.WindowsSdkVersion: + intver = ver[:ver.rfind('.')] + path = r'Microsoft SDKs\Windows Kits\%s' % (intver) + d = os.path.join(self.ProgramFiles, path) + if os.path.isdir(d): + sdkdir = d + if not sdkdir or not os.path.isdir(sdkdir): + # If fail, use default old path + for ver in self.WindowsSdkVersion: + path = r'Microsoft SDKs\Windows\v%s' % ver + d = os.path.join(self.ProgramFiles, path) + if os.path.isdir(d): + sdkdir = d + if not sdkdir: + # If fail, use Platform SDK + sdkdir = os.path.join(self.VCInstallDir, 'PlatformSDK') + return sdkdir + + @property + def WindowsSDKExecutablePath(self): + """ + Microsoft Windows SDK executable directory. + """ + # Find WinSDK NetFx Tools registry dir name + if self.vc_ver <= 11.0: + netfxver = 35 + arch = '' + else: + netfxver = 40 + hidex86 = True if self.vc_ver <= 12.0 else False + arch = self.pi.current_dir(x64=True, hidex86=hidex86) + fx = 'WinSDK-NetFx%dTools%s' % (netfxver, arch.replace('\\', '-')) + + # liste all possibles registry paths + regpaths = [] + if self.vc_ver >= 14.0: + for ver in self.NetFxSdkVersion: + regpaths += [os.path.join(self.ri.netfx_sdk, ver, fx)] + + for ver in self.WindowsSdkVersion: + regpaths += [os.path.join(self.ri.windows_sdk, 'v%sA' % ver, fx)] + + # Return installation folder from the more recent path + for path in regpaths: + execpath = self.ri.lookup(path, 'installationfolder') + if execpath: + break + return execpath + + @property + def FSharpInstallDir(self): + """ + Microsoft Visual F# directory. + """ + path = r'%0.1f\Setup\F#' % self.vc_ver + path = os.path.join(self.ri.visualstudio, path) + return self.ri.lookup(path, 'productdir') or '' + + @property + def UniversalCRTSdkDir(self): + """ + Microsoft Universal CRT SDK directory. + """ + # Set Kit Roots versions for specified MSVC++ version + if self.vc_ver >= 14.0: + vers = ('10', '81') + else: + vers = () + + # Find path of the more recent Kit + for ver in vers: + sdkdir = self.ri.lookup(self.ri.windows_kits_roots, + 'kitsroot%s' % ver) + if sdkdir: + break + return sdkdir or '' + + @property + def UniversalCRTSdkLastVersion(self): + """ + Microsoft Universal C Runtime SDK last version + """ + return self._use_last_dir_name(os.path.join( + self.UniversalCRTSdkDir, 'lib')) + + @property + def NetFxSdkVersion(self): + """ + Microsoft .NET Framework SDK versions. + """ + # Set FxSdk versions for specified MSVC++ version + if self.vc_ver >= 14.0: + return ('4.6.1', '4.6') + else: + return () + + @property + def NetFxSdkDir(self): + """ + Microsoft .NET Framework SDK directory. + """ + for ver in self.NetFxSdkVersion: + loc = os.path.join(self.ri.netfx_sdk, ver) + sdkdir = self.ri.lookup(loc, 'kitsinstallationfolder') _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit