Author: Philip Jenvey <pjen...@underboss.org> Branch: py3k Changeset: r75210:ace68169b31e Date: 2015-01-02 12:39 -0800 http://bitbucket.org/pypy/pypy/changeset/ace68169b31e/
Log: merge default diff too long, truncating to 2000 out of 15159 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2014 +PyPy Copyright holders 2003-2015 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py --- a/lib-python/2.7/distutils/unixccompiler.py +++ b/lib-python/2.7/distutils/unixccompiler.py @@ -58,7 +58,7 @@ executables = {'preprocessor' : None, 'compiler' : ["cc"], 'compiler_so' : ["cc"], - 'compiler_cxx' : ["cc"], + 'compiler_cxx' : ["c++"], # pypy: changed, 'cc' is bogus 'linker_so' : ["cc", "-shared"], 'linker_exe' : ["cc"], 'archiver' : ["ar", "-cr"], diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py --- a/lib-python/2.7/sqlite3/test/dbapi.py +++ b/lib-python/2.7/sqlite3/test/dbapi.py @@ -478,6 +478,29 @@ except TypeError: pass + def CheckCurDescription(self): + self.cu.execute("select * from test") + + actual = self.cu.description + expected = [ + ('id', None, None, None, None, None, None), + ('name', None, None, None, None, None, None), + ('income', None, None, None, None, None, None), + ] + self.assertEqual(expected, actual) + + def CheckCurDescriptionVoidStatement(self): + self.cu.execute("insert into test(name) values (?)", ("foo",)) + self.assertIsNone(self.cu.description) + + def CheckCurDescriptionWithoutStatement(self): + cu = self.cx.cursor() + try: + self.assertIsNone(cu.description) + finally: + cu.close() + + @unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): def setUp(self): diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -1589,7 +1589,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy-c.so']: + for libname in ['libpypy-c.so', 'libpypy-c.dylib']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py --- a/lib-python/2.7/test/test_collections.py +++ b/lib-python/2.7/test/test_collections.py @@ -1108,6 +1108,16 @@ od.popitem() self.assertEqual(len(od), 0) + def test_popitem_first(self): + pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] + shuffle(pairs) + od = OrderedDict(pairs) + while pairs: + self.assertEqual(od.popitem(last=False), pairs.pop(0)) + with self.assertRaises(KeyError): + od.popitem(last=False) + self.assertEqual(len(od), 0) + def test_pop(self): pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) @@ -1179,7 +1189,11 @@ od = OrderedDict(pairs) # yaml.dump(od) --> # '!!python/object/apply:__main__.OrderedDict\n- - [a, 1]\n - [b, 2]\n' - self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1])) + + # PyPy bug fix: added [0] at the end of this line, because the + # test is really about the 2-tuples that need to be 2-lists + # inside the list of 6 of them + self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1][0])) def test_reduce_not_too_fat(self): # do not save instance dictionary if not needed @@ -1189,6 +1203,16 @@ od.x = 10 self.assertEqual(len(od.__reduce__()), 3) + def test_reduce_exact_output(self): + # PyPy: test that __reduce__() produces the exact same answer as + # CPython does, even though in the 'all_ordered_dicts' branch we + # have to emulate it. + pairs = [['c', 1], ['b', 2], ['d', 4]] + od = OrderedDict(pairs) + self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,))) + od.x = 10 + self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,), {'x': 10})) + def test_repr(self): od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]) self.assertEqual(repr(od), diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py --- a/lib-python/2.7/test/test_xml_etree.py +++ b/lib-python/2.7/test/test_xml_etree.py @@ -225,9 +225,9 @@ >>> element.remove(subelement) >>> serialize(element) # 5 '<tag key="value" />' - >>> element.remove(subelement) + >>> element.remove(subelement) # doctest: +ELLIPSIS Traceback (most recent call last): - ValueError: list.remove(x): x not in list + ValueError: list.remove(... >>> serialize(element) # 6 '<tag key="value" />' >>> element[0:0] = [subelement, subelement, subelement] diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt --- a/lib-python/stdlib-upgrade.txt +++ b/lib-python/stdlib-upgrade.txt @@ -7,7 +7,7 @@ 1. check out the branch vendor/stdlib 2. upgrade the files there -3. update stdlib-versions.txt with the output of hg -id from the cpython repo +3. update stdlib-version.txt with the output of hg -id from the cpython repo 4. commit 5. update to default/py3k 6. create a integration branch for the new stdlib diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py --- a/lib_pypy/_functools.py +++ b/lib_pypy/_functools.py @@ -33,7 +33,10 @@ of the given arguments and keywords. """ - def __init__(self, func, *args, **keywords): + def __init__(self, *args, **keywords): + if not args: + raise TypeError('__init__() takes at least 2 arguments (1 given)') + func, args = args[0], args[1:] if not callable(func): raise TypeError("the first argument must be callable") self._func = func diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -1187,8 +1187,9 @@ try: return self.__description except AttributeError: - self.__description = self.__statement._get_description() - return self.__description + if self.__statement: + self.__description = self.__statement._get_description() + return self.__description description = property(__get_description) def __get_lastrowid(self): diff --git a/lib_pypy/readline.py b/lib_pypy/readline.py --- a/lib_pypy/readline.py +++ b/lib_pypy/readline.py @@ -8,5 +8,9 @@ try: from pyrepl.readline import * -except SyntaxError: - raise ImportError +except ImportError: + import sys + if sys.platform == 'win32': + raise ImportError("the 'readline' module is not available on Windows" + " (on either PyPy or CPython)") + raise diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst --- a/pypy/doc/embedding.rst +++ b/pypy/doc/embedding.rst @@ -6,6 +6,10 @@ C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_ project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API. +**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default +on linux, linux64 and windows. We will make it the default on all platforms +by the time of the next release. + The first thing that you need is to compile PyPy yourself with the option ``--shared``. We plan to make ``--shared`` the default in the future. Consult the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so`` @@ -93,12 +97,18 @@ return res; } -If we save it as ``x.c`` now, compile it and run it with:: +If we save it as ``x.c`` now, compile it and run it (on linux) with:: fijal@hermann:/opt/pypy$ gcc -o x x.c -lpypy-c -L. fijal@hermann:/opt/pypy$ LD_LIBRARY_PATH=. ./x hello from pypy +on OSX it is necessary to set the rpath of the binary if one wants to link to it:: + + gcc -o x x.c -lpypy-c -L. -Wl,-rpath -Wl,@executable_path + ./x + hello from pypy + Worked! .. note:: If the compilation fails because of missing PyPy.h header file, diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -35,6 +35,13 @@ PyPy's bytearray type is very inefficient. It would be an interesting task to look into possible optimizations on this. +Implement AF_XXX packet types for PyPy +-------------------------------------- + +PyPy is missing AF_XXX types of sockets. Implementing it is easy-to-medium +task. `bug report`_ + +.. _`bug report`: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets#more Implement copy-on-write list slicing ------------------------------------ diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -241,8 +241,7 @@ enable_translationmodules(config) config.translation.suggest(check_str_without_nul=True) - if sys.platform.startswith('linux'): - config.translation.suggest(shared=True) + config.translation.suggest(shared=True) if config.translation.thread: config.objspace.usemodules.thread = True diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py --- a/pypy/interpreter/astcompiler/optimize.py +++ b/pypy/interpreter/astcompiler/optimize.py @@ -89,17 +89,16 @@ class __extend__(ast.BoolOp): - def _accept_jump_if_any_is(self, gen, condition, target): - self.values[0].accept_jump_if(gen, condition, target) - for i in range(1, len(self.values)): + def _accept_jump_if_any_is(self, gen, condition, target, skip_last=0): + for i in range(len(self.values) - skip_last): self.values[i].accept_jump_if(gen, condition, target) def accept_jump_if(self, gen, condition, target): if condition and self.op == ast.And or \ (not condition and self.op == ast.Or): end = gen.new_block() - self._accept_jump_if_any_is(gen, not condition, end) - gen.emit_jump(ops.JUMP_FORWARD, target) + self._accept_jump_if_any_is(gen, not condition, end, skip_last=1) + self.values[-1].accept_jump_if(gen, condition, target) gen.use_next_block(end) else: self._accept_jump_if_any_is(gen, condition, target) diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -125,6 +125,8 @@ else: return self.space.builtin + _NO_CELLS = [] + @jit.unroll_safe def initialize_frame_scopes(self, outer_func, code): # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS. @@ -143,7 +145,7 @@ nfreevars = len(code.co_freevars) if not nfreevars: if not ncellvars: - self.cells = [] + self.cells = self._NO_CELLS return # no self.cells needed - fast path elif outer_func is None: space = self.space diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -34,6 +34,7 @@ 'newp_handle': 'handle.newp_handle', 'from_handle': 'handle.from_handle', '_get_types': 'func._get_types', + 'from_buffer': 'func.from_buffer', 'string': 'func.string', 'buffer': 'cbuffer.buffer', diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -440,6 +440,25 @@ return "handle to %s" % (self.space.str_w(w_repr),) +class W_CDataFromBuffer(W_CData): + _attrs_ = ['buf', 'length', 'w_keepalive'] + _immutable_fields_ = ['buf', 'length', 'w_keepalive'] + + def __init__(self, space, cdata, ctype, buf, w_object): + W_CData.__init__(self, space, cdata, ctype) + self.buf = buf + self.length = buf.getlength() + self.w_keepalive = w_object + + def get_array_length(self): + return self.length + + def _repr_extra(self): + w_repr = self.space.repr(self.w_keepalive) + return "buffer len %d from '%s' object" % ( + self.length, self.space.type(self.w_keepalive).name) + + W_CData.typedef = TypeDef( '_cffi_backend.CData', __module__ = '_cffi_backend', diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py --- a/pypy/module/_cffi_backend/func.py +++ b/pypy/module/_cffi_backend/func.py @@ -76,3 +76,32 @@ def _get_types(space): return space.newtuple([space.gettypefor(cdataobj.W_CData), space.gettypefor(ctypeobj.W_CType)]) + +# ____________________________________________________________ + +@unwrap_spec(w_ctype=ctypeobj.W_CType) +def from_buffer(space, w_ctype, w_x): + from pypy.module._cffi_backend import ctypearray, ctypeprim + # + if (not isinstance(w_ctype, ctypearray.W_CTypeArray) or + not isinstance(w_ctype.ctptr.ctitem, ctypeprim.W_CTypePrimitiveChar)): + raise oefmt(space.w_TypeError, + "needs 'char[]', got '%s'", w_ctype.name) + # + # xxx do we really need to implement the same mess as in CPython 2.7 + # w.r.t. buffers and memoryviews?? + try: + buf = space.readbuf_w(w_x) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + buf = space.buffer_w(w_x, space.BUF_SIMPLE) + try: + _cdata = buf.get_raw_address() + except ValueError: + raise oefmt(space.w_TypeError, + "from_buffer() got a '%T' object, which supports the " + "buffer interface but cannot be rendered as a plain " + "raw address on PyPy", w_x) + # + return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x) diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -62,10 +62,54 @@ eptype("intptr_t", rffi.INTPTR_T, ctypeprim.W_CTypePrimitiveSigned) eptype("uintptr_t", rffi.UINTPTR_T, ctypeprim.W_CTypePrimitiveUnsigned) -eptype("ptrdiff_t", rffi.INTPTR_T, ctypeprim.W_CTypePrimitiveSigned) # <-xxx eptype("size_t", rffi.SIZE_T, ctypeprim.W_CTypePrimitiveUnsigned) eptype("ssize_t", rffi.SSIZE_T, ctypeprim.W_CTypePrimitiveSigned) +_WCTSigned = ctypeprim.W_CTypePrimitiveSigned +_WCTUnsign = ctypeprim.W_CTypePrimitiveUnsigned + +eptype("ptrdiff_t", getattr(rffi, 'PTRDIFF_T', rffi.INTPTR_T), _WCTSigned) +eptype("intmax_t", getattr(rffi, 'INTMAX_T', rffi.LONGLONG), _WCTSigned) +eptype("uintmax_t", getattr(rffi, 'UINTMAX_T', rffi.LONGLONG), _WCTUnsign) + +if hasattr(rffi, 'INT_LEAST8_T'): + eptype("int_least8_t", rffi.INT_LEAST8_T, _WCTSigned) + eptype("int_least16_t", rffi.INT_LEAST16_T, _WCTSigned) + eptype("int_least32_t", rffi.INT_LEAST32_T, _WCTSigned) + eptype("int_least64_t", rffi.INT_LEAST64_T, _WCTSigned) + eptype("uint_least8_t", rffi.UINT_LEAST8_T, _WCTUnsign) + eptype("uint_least16_t",rffi.UINT_LEAST16_T, _WCTUnsign) + eptype("uint_least32_t",rffi.UINT_LEAST32_T, _WCTUnsign) + eptype("uint_least64_t",rffi.UINT_LEAST64_T, _WCTUnsign) +else: + eptypesize("int_least8_t", 1, _WCTSigned) + eptypesize("uint_least8_t", 1, _WCTUnsign) + eptypesize("int_least16_t", 2, _WCTSigned) + eptypesize("uint_least16_t", 2, _WCTUnsign) + eptypesize("int_least32_t", 4, _WCTSigned) + eptypesize("uint_least32_t", 4, _WCTUnsign) + eptypesize("int_least64_t", 8, _WCTSigned) + eptypesize("uint_least64_t", 8, _WCTUnsign) + +if hasattr(rffi, 'INT_FAST8_T'): + eptype("int_fast8_t", rffi.INT_FAST8_T, _WCTSigned) + eptype("int_fast16_t", rffi.INT_FAST16_T, _WCTSigned) + eptype("int_fast32_t", rffi.INT_FAST32_T, _WCTSigned) + eptype("int_fast64_t", rffi.INT_FAST64_T, _WCTSigned) + eptype("uint_fast8_t", rffi.UINT_FAST8_T, _WCTUnsign) + eptype("uint_fast16_t",rffi.UINT_FAST16_T, _WCTUnsign) + eptype("uint_fast32_t",rffi.UINT_FAST32_T, _WCTUnsign) + eptype("uint_fast64_t",rffi.UINT_FAST64_T, _WCTUnsign) +else: + eptypesize("int_fast8_t", 1, _WCTSigned) + eptypesize("uint_fast8_t", 1, _WCTUnsign) + eptypesize("int_fast16_t", 2, _WCTSigned) + eptypesize("uint_fast16_t", 2, _WCTUnsign) + eptypesize("int_fast32_t", 4, _WCTSigned) + eptypesize("uint_fast32_t", 4, _WCTUnsign) + eptypesize("int_fast64_t", 8, _WCTSigned) + eptypesize("uint_fast64_t", 8, _WCTUnsign) + @unwrap_spec(name=str) def new_primitive_type(space, name): try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -397,7 +397,7 @@ def test_invalid_indexing(): p = new_primitive_type("int") x = cast(p, 42) - py.test.raises(TypeError, "p[0]") + py.test.raises(TypeError, "x[0]") def test_default_str(): BChar = new_primitive_type("char") @@ -2718,7 +2718,16 @@ def test_nonstandard_integer_types(): for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t', 'uint32_t', 'int64_t', 'uint64_t', 'intptr_t', - 'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t']: + 'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t', + 'int_least8_t', 'uint_least8_t', + 'int_least16_t', 'uint_least16_t', + 'int_least32_t', 'uint_least32_t', + 'int_least64_t', 'uint_least64_t', + 'int_fast8_t', 'uint_fast8_t', + 'int_fast16_t', 'uint_fast16_t', + 'int_fast32_t', 'uint_fast32_t', + 'int_fast64_t', 'uint_fast64_t', + 'intmax_t', 'uintmax_t']: new_primitive_type(typename) # works def test_cannot_convert_unicode_to_charp(): @@ -3186,6 +3195,20 @@ ('a2', BChar, 5)], None, -1, -1, SF_PACKED) +def test_from_buffer(): + import array + a = array.array('H', [10000, 20000, 30000]) + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + c = from_buffer(BCharA, a) + assert typeof(c) is BCharA + assert len(c) == 6 + assert repr(c) == "<cdata 'char[]' buffer len 6 from 'array.array' object>" + p = new_pointer_type(new_primitive_type("unsigned short")) + cast(p, c)[1] += 500 + assert list(a) == [10000, 20500, 30000] + def test_version(): # this test is here mostly for PyPy assert __version__ == "0.8.6" diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py --- a/pypy/module/_cffi_backend/test/test_c.py +++ b/pypy/module/_cffi_backend/test/test_c.py @@ -31,7 +31,7 @@ class AppTestC(object): """Populated below, hack hack hack.""" - spaceconfig = dict(usemodules=('_cffi_backend', '_io')) + spaceconfig = dict(usemodules=('_cffi_backend', '_io', 'array')) def setup_class(cls): testfuncs_w = [] diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -564,7 +564,7 @@ # Flush the write buffer if necessary if self.writable: - self._writer_flush_unlocked(space) + self._flush_and_rewind_unlocked(space) self._reader_reset_buf() # Read whole blocks, and don't buffer them diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py --- a/pypy/module/_io/test/test_io.py +++ b/pypy/module/_io/test/test_io.py @@ -400,3 +400,32 @@ f.read(1) f.seek(-1, 1) f.write(b'') + + def test_issue1902_2(self): + import _io + with _io.open(self.tmpfile, 'w+b', 4096) as f: + f.write(b'\xff' * 13569) + f.flush() + f.seek(0, 0) + + f.read(1) + f.seek(-1, 1) + f.write(b'\xff') + f.seek(1, 0) + f.read(4123) + f.seek(-4123, 1) + + def test_issue1902_3(self): + import _io + buffer_size = 4096 + with _io.open(self.tmpfile, 'w+b', buffer_size) as f: + f.write(b'\xff' * buffer_size * 3) + f.flush() + f.seek(0, 0) + + f.read(1) + f.seek(-1, 1) + f.write(b'\xff') + f.seek(1, 0) + f.read(buffer_size * 2) + assert f.tell() == 1 + buffer_size * 2 diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py --- a/pypy/module/_rawffi/alt/interp_funcptr.py +++ b/pypy/module/_rawffi/alt/interp_funcptr.py @@ -14,6 +14,7 @@ from rpython.rlib.objectmodel import we_are_translated from pypy.module._rawffi.alt.type_converter import FromAppLevelConverter, ToAppLevelConverter from pypy.module._rawffi.interp_rawffi import got_libffi_error, wrap_dlopenerror +from pypy.module._rawffi import lasterror import os if os.name == 'nt': @@ -202,11 +203,23 @@ self.func = func self.argchain = argchain + def before(self): + lasterror.restore_last_error(self.space) + + def after(self): + lasterror.save_last_error(self.space) + def get_longlong(self, w_ffitype): - return self.func.call(self.argchain, rffi.LONGLONG) + self.before() + x = self.func.call(self.argchain, rffi.LONGLONG) + self.after() + return x def get_ulonglong(self, w_ffitype): - return self.func.call(self.argchain, rffi.ULONGLONG) + self.before() + x = self.func.call(self.argchain, rffi.ULONGLONG) + self.after() + return x def get_signed(self, w_ffitype): # if the declared return type of the function is smaller than LONG, @@ -217,64 +230,94 @@ # to space.wrap in order to get a nice applevel <int>. # restype = w_ffitype.get_ffitype() + self.before() call = self.func.call if restype is libffi.types.slong: - return call(self.argchain, rffi.LONG) + x = call(self.argchain, rffi.LONG) elif restype is libffi.types.sint: - return rffi.cast(rffi.LONG, call(self.argchain, rffi.INT)) + x = rffi.cast(rffi.LONG, call(self.argchain, rffi.INT)) elif restype is libffi.types.sshort: - return rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT)) + x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT)) elif restype is libffi.types.schar: - return rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR)) + x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR)) else: - self.error(w_ffitype) + raise self.error(w_ffitype) + self.after() + return x def get_unsigned(self, w_ffitype): - return self.func.call(self.argchain, rffi.ULONG) + self.before() + x = self.func.call(self.argchain, rffi.ULONG) + self.after() + return x def get_unsigned_which_fits_into_a_signed(self, w_ffitype): # the same comment as get_signed apply restype = w_ffitype.get_ffitype() + self.before() call = self.func.call if restype is libffi.types.uint: assert not libffi.IS_32_BIT # on 32bit machines, we should never get here, because it's a case # which has already been handled by get_unsigned above. - return rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT)) + x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT)) elif restype is libffi.types.ushort: - return rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT)) + x = rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT)) elif restype is libffi.types.uchar: - return rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR)) + x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR)) else: - self.error(w_ffitype) + raise self.error(w_ffitype) + self.after() + return x def get_pointer(self, w_ffitype): + self.before() ptrres = self.func.call(self.argchain, rffi.VOIDP) + self.after() return rffi.cast(rffi.ULONG, ptrres) def get_char(self, w_ffitype): - return self.func.call(self.argchain, rffi.UCHAR) + self.before() + x = self.func.call(self.argchain, rffi.UCHAR) + self.after() + return x def get_unichar(self, w_ffitype): - return self.func.call(self.argchain, rffi.WCHAR_T) + self.before() + x = self.func.call(self.argchain, rffi.WCHAR_T) + self.after() + return x def get_float(self, w_ffitype): - return self.func.call(self.argchain, rffi.DOUBLE) + self.before() + x = self.func.call(self.argchain, rffi.DOUBLE) + self.after() + return x def get_singlefloat(self, w_ffitype): - return self.func.call(self.argchain, rffi.FLOAT) + self.before() + x = self.func.call(self.argchain, rffi.FLOAT) + self.after() + return x def get_struct(self, w_ffitype, w_structdescr): + self.before() addr = self.func.call(self.argchain, rffi.LONG, is_struct=True) + self.after() return w_structdescr.fromaddress(self.space, addr) def get_struct_rawffi(self, w_ffitype, w_structdescr): + self.before() uintval = self.func.call(self.argchain, rffi.ULONG, is_struct=True) + self.after() return w_structdescr.fromaddress(self.space, uintval) def get_void(self, w_ffitype): - return self.func.call(self.argchain, lltype.Void) + self.before() + x = self.func.call(self.argchain, lltype.Void) + self.after() + return x def unpack_argtypes(space, w_argtypes, w_restype): diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py --- a/pypy/module/_rawffi/buffer.py +++ b/pypy/module/_rawffi/buffer.py @@ -1,4 +1,5 @@ from rpython.rlib.buffer import Buffer +from rpython.rtyper.lltypesystem import rffi # XXX not the most efficient implementation @@ -20,3 +21,7 @@ def setitem(self, index, char): ll_buffer = self.datainstance.ll_buffer ll_buffer[index] = char + + def get_raw_address(self): + ll_buffer = self.datainstance.ll_buffer + return rffi.cast(rffi.CCHARP, ll_buffer) diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -20,6 +20,7 @@ from rpython.rlib.rarithmetic import intmask, r_uint from pypy.module._rawffi.buffer import RawFFIBuffer from pypy.module._rawffi.tracker import tracker +from pypy.module._rawffi import lasterror TYPEMAP = { # XXX A mess with unsigned/signed/normal chars :-/ @@ -498,10 +499,14 @@ try: if self.resshape is not None: result = self.resshape.allocate(space, 1, autofree=True) + lasterror.restore_last_error(space) self.ptr.call(args_ll, result.ll_buffer) + lasterror.save_last_error(space) return space.wrap(result) else: + lasterror.restore_last_error(space) self.ptr.call(args_ll, lltype.nullptr(rffi.VOIDP.TO)) + lasterror.save_last_error(space) return space.w_None except StackCheckError, e: raise OperationError(space.w_ValueError, space.wrap(e.message)) @@ -618,12 +623,10 @@ if sys.platform == 'win32': def get_last_error(space): - from rpython.rlib.rwin32 import GetLastError - return space.wrap(GetLastError()) + return space.wrap(lasterror.fetch_last_error(space)) @unwrap_spec(error=int) def set_last_error(space, error): - from rpython.rlib.rwin32 import SetLastError - SetLastError(error) + lasterror.store_last_error(space, error) else: # always have at least a dummy version of these functions # (https://bugs.pypy.org/issue1242) diff --git a/pypy/module/_rawffi/lasterror.py b/pypy/module/_rawffi/lasterror.py new file mode 100644 --- /dev/null +++ b/pypy/module/_rawffi/lasterror.py @@ -0,0 +1,40 @@ +# For Windows only. +# https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror + +import os + +_MS_WINDOWS = os.name == "nt" + + +if _MS_WINDOWS: + from rpython.rlib import rwin32 + from pypy.interpreter.executioncontext import ExecutionContext + + + ExecutionContext._rawffi_last_error = 0 + + def fetch_last_error(space): + ec = space.getexecutioncontext() + return ec._rawffi_last_error + + def store_last_error(space, last_error): + ec = space.getexecutioncontext() + ec._rawffi_last_error = last_error + + def restore_last_error(space): + ec = space.getexecutioncontext() + lasterror = ec._rawffi_last_error + rwin32.SetLastError(lasterror) + + def save_last_error(space): + lasterror = rwin32.GetLastError() + ec = space.getexecutioncontext() + ec._rawffi_last_error = lasterror + +else: + + def restore_last_error(space): + pass + + def save_last_error(space): + pass diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -16,6 +16,7 @@ #include "src/precommondefs.h" #include <stdlib.h> #include <stdio.h> + #include <errno.h> struct x { @@ -204,6 +205,24 @@ return inp; } + RPY_EXPORTED + int check_errno(int incoming) + { + int old_errno = errno; + errno = incoming; + return old_errno; + } + + #ifdef _WIN32 + #include <Windows.h> + RPY_EXPORTED + int check_last_error(int incoming) + { + int old_errno = GetLastError(); + SetLastError(incoming); + return old_errno; + } + #endif ''')) eci = ExternalCompilationInfo(include_dirs=[cdir]) return str(platform.compile([c_file], eci, 'x', standalone=False)) @@ -1118,6 +1137,15 @@ b[3] = b'x' assert b[3] == b'x' + def test_pypy_raw_address(self): + import _rawffi + S = _rawffi.Structure((40, 1)) + s = S(autofree=True) + addr = buffer(s)._pypy_raw_address() + assert type(addr) is int + assert buffer(s)._pypy_raw_address() == addr + assert buffer(s, 10)._pypy_raw_address() == addr + 10 + def test_union(self): import _rawffi longsize = _rawffi.sizeof('l') @@ -1143,6 +1171,37 @@ raises(OverflowError, "arg1[0] = 10**900") arg1.free() + def test_errno(self): + import _rawffi + lib = _rawffi.CDLL(self.lib_name) + A = _rawffi.Array('i') + f = lib.ptr('check_errno', ['i'], 'i') + _rawffi.set_errno(42) + arg = A(1) + arg[0] = 43 + res = f(arg) + assert res[0] == 42 + z = _rawffi.get_errno() + assert z == 43 + arg.free() + + def test_last_error(self): + import sys + if sys.platform != 'win32': + skip("Windows test") + import _rawffi + lib = _rawffi.CDLL(self.lib_name) + A = _rawffi.Array('i') + f = lib.ptr('check_last_error', ['i'], 'i') + _rawffi.set_last_error(42) + arg = A(1) + arg[0] = 43 + res = f(arg) + assert res[0] == 42 + z = _rawffi.get_last_error() + assert z == 43 + arg.free() + def test_char_array_int(self): import _rawffi A = _rawffi.Array('c') diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -55,7 +55,8 @@ if not OPENSSL_NO_SSL2: constants["PROTOCOL_SSLv2"] = PY_SSL_VERSION_SSL2 -constants["PROTOCOL_SSLv3"] = PY_SSL_VERSION_SSL3 +if not OPENSSL_NO_SSL3: + constants["PROTOCOL_SSLv3"] = PY_SSL_VERSION_SSL3 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23 constants["PROTOCOL_TLSv1"] = PY_SSL_VERSION_TLS1 @@ -95,7 +96,7 @@ def __init__(self, space, protocol): if protocol == PY_SSL_VERSION_TLS1: method = libssl_TLSv1_method() - elif protocol == PY_SSL_VERSION_SSL3: + elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3: method = libssl_SSLv3_method() elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2: method = libssl_SSLv2_method() diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -58,7 +58,7 @@ w_globals = from_ref(space, py_frame.c_f_globals) frame = space.FrameClass(space, code, w_globals, outer_func=None) - frame.f_lineno = py_frame.c_f_lineno + frame.f_lineno = rffi.getintfield(py_frame, 'c_f_lineno') w_obj = space.wrap(frame) track_reference(space, py_obj, w_obj) return w_obj diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -244,6 +244,9 @@ def getitem(self, index): return self.ptr[index] + def get_raw_address(self): + return rffi.cast(rffi.CCHARP, self.ptr) + def wrap_getbuffer(space, w_self, w_args, func): func_target = rffi.cast(getbufferproc, func) with lltype.scoped_alloc(Py_buffer) as view: diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -30,6 +30,7 @@ 'get_referrers': 'referents.get_referrers', '_dump_rpy_heap': 'referents._dump_rpy_heap', 'get_typeids_z': 'referents.get_typeids_z', + 'get_typeids_list': 'referents.get_typeids_list', 'GcRef': 'referents.W_GcRef', }) MixedModule.__init__(self, space, w_name) diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py --- a/pypy/module/gc/app_referents.py +++ b/pypy/module/gc/app_referents.py @@ -16,7 +16,8 @@ [0][0][0][-1] inserted after all GC roots, before all non-roots. If the argument is a filename and the 'zlib' module is available, - we also write a 'typeids.txt' in the same directory, if none exists. + we also write 'typeids.txt' and 'typeids.lst' in the same directory, + if they don't already exist. """ if isinstance(file, str): f = open(file, 'wb') @@ -30,7 +31,13 @@ filename2 = os.path.join(os.path.dirname(file), 'typeids.txt') if not os.path.exists(filename2): data = zlib.decompress(gc.get_typeids_z()) - f = open(filename2, 'wb') + f = open(filename2, 'w') + f.write(data) + f.close() + filename2 = os.path.join(os.path.dirname(file), 'typeids.lst') + if not os.path.exists(filename2): + data = ''.join(['%d\n' % n for n in gc.get_typeids_list()]) + f = open(filename2, 'w') f.write(data) f.close() else: diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py --- a/pypy/module/gc/referents.py +++ b/pypy/module/gc/referents.py @@ -228,3 +228,8 @@ a = rgc.get_typeids_z() s = ''.join([a[i] for i in range(len(a))]) return space.wrap(s) + +def get_typeids_list(space): + l = rgc.get_typeids_list() + list_w = [space.wrap(l[i]) for i in range(len(l))] + return space.newlist(list_w) diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -1,7 +1,7 @@ from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.error import OperationError +from pypy.interpreter.error import OperationError, oefmt from rpython.tool.pairtype import extendabletype - +from pypy.module.micronumpy import support def wrap_impl(space, w_cls, w_instance, impl): if w_cls is None or space.is_w(w_cls, space.gettypefor(W_NDimArray)): @@ -44,11 +44,32 @@ return W_NDimArray(impl) @staticmethod - def from_shape_and_storage(space, shape, storage, dtype, order='C', owning=False, - w_subtype=None, w_base=None, writable=True): + def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1, + order='C', owning=False, w_subtype=None, + w_base=None, writable=True, strides=None): from pypy.module.micronumpy import concrete - from pypy.module.micronumpy.strides import calc_strides - strides, backstrides = calc_strides(shape, dtype, order) + from pypy.module.micronumpy.strides import (calc_strides, + calc_backstrides) + isize = dtype.elsize + if storage_bytes > 0 : + totalsize = support.product(shape) * isize + if totalsize > storage_bytes: + raise OperationError(space.w_TypeError, space.wrap( + "buffer is too small for requested array")) + else: + storage_bytes = support.product(shape) * isize + if strides is None: + strides, backstrides = calc_strides(shape, dtype, order) + else: + if len(strides) != len(shape): + raise oefmt(space.w_ValueError, + 'strides, if given, must be the same length as shape') + for i in range(len(strides)): + if strides[i] < 0 or strides[i]*shape[i] > storage_bytes: + raise oefmt(space.w_ValueError, + 'strides is incompatible with shape of requested ' + 'array and size of buffer') + backstrides = calc_backstrides(strides, shape) if w_base is not None: if owning: raise OperationError(space.w_ValueError, diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py --- a/pypy/module/micronumpy/boxes.py +++ b/pypy/module/micronumpy/boxes.py @@ -59,9 +59,9 @@ _mixin_ = True def reduce(self, space): - numpypy = space.getbuiltinmodule("_numpypy") - assert isinstance(numpypy, MixedModule) - multiarray = numpypy.get("multiarray") + _numpypy = space.getbuiltinmodule("_numpypy") + assert isinstance(_numpypy, MixedModule) + multiarray = _numpypy.get("multiarray") assert isinstance(multiarray, MixedModule) scalar = multiarray.get("scalar") @@ -167,7 +167,7 @@ if len(args_w) >= 1: for w_arg in args_w: try: - idx = support.index_w(space, w_arg) + support.index_w(space, w_arg) except OperationError: raise oefmt(space.w_TypeError, "an integer is required") raise oefmt(space.w_ValueError, "axes don't match array") diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -34,8 +34,8 @@ SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", - "unegative", "flat", "tostring","count_nonzero", - "argsort"] + "unegative", "flat", "tostring", "count_nonzero", + "argsort", "cumsum", "logical_xor_reduce"] TWO_ARG_FUNCTIONS = ["dot", 'take', 'searchsorted'] TWO_ARG_FUNCTIONS_OR_NONE = ['view', 'astype'] THREE_ARG_FUNCTIONS = ['where'] @@ -559,6 +559,11 @@ w_res = arr.descr_any(interp.space) elif self.name == "all": w_res = arr.descr_all(interp.space) + elif self.name == "cumsum": + w_res = arr.descr_cumsum(interp.space) + elif self.name == "logical_xor_reduce": + logical_xor = ufuncs.get(interp.space).logical_xor + w_res = logical_xor.reduce(interp.space, arr, None) elif self.name == "unegative": neg = ufuncs.get(interp.space).negative w_res = neg.call(interp.space, [arr]) diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -11,7 +11,7 @@ from pypy.module.micronumpy.iterators import ArrayIter from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk, RecordChunk, calc_strides, calc_new_strides, shape_agreement, - calculate_broadcast_strides) + calculate_broadcast_strides, calc_backstrides) class BaseConcreteArray(object): @@ -79,10 +79,7 @@ self.get_strides(), self.order) if new_strides is not None: # We can create a view, strides somehow match up. - ndims = len(new_shape) - new_backstrides = [0] * ndims - for nd in range(ndims): - new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd] + new_backstrides = calc_backstrides(new_strides, new_shape) assert isinstance(orig_array, W_NDimArray) or orig_array is None return SliceArray(self.start, new_strides, new_backstrides, new_shape, self, orig_array) diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -302,5 +302,5 @@ return a else: writable = not buf.readonly - return W_NDimArray.from_shape_and_storage(space, [n], storage, dtype=dtype, - w_base=w_buffer, writable=writable) + return W_NDimArray.from_shape_and_storage(space, [n], storage, storage_bytes=s, + dtype=dtype, w_base=w_buffer, writable=writable) diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -286,7 +286,6 @@ def descr_hash(self, space): return space.wrap(self._compute_hash(space, 0x345678)) - def descr_str(self, space): if self.fields: return space.str(self.descr_get_descr(space)) @@ -394,7 +393,7 @@ alignment = space.int_w(space.getitem(w_data, space.wrap(6))) if (w_names == space.w_None) != (w_fields == space.w_None): - raise oefmt(space.w_ValueError, "inconsistent fields and names") + raise oefmt(space.w_ValueError, "inconsistent fields and names in Numpy dtype unpickling") self.byteorder = endian self.shape = [] diff --git a/pypy/module/micronumpy/flagsobj.py b/pypy/module/micronumpy/flagsobj.py --- a/pypy/module/micronumpy/flagsobj.py +++ b/pypy/module/micronumpy/flagsobj.py @@ -1,47 +1,30 @@ +from rpython.rlib import jit + from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.module.micronumpy import constants as NPY - +from pypy.module.micronumpy.strides import is_c_contiguous, is_f_contiguous def enable_flags(arr, flags): arr.flags |= flags - def clear_flags(arr, flags): arr.flags &= ~flags - def _update_contiguous_flags(arr): - shape = arr.shape - strides = arr.strides - - is_c_contig = True - sd = arr.dtype.elsize - for i in range(len(shape) - 1, -1, -1): - dim = shape[i] - if strides[i] != sd: - is_c_contig = False - break - if dim == 0: - break - sd *= dim + is_c_contig = is_c_contiguous(arr) if is_c_contig: enable_flags(arr, NPY.ARRAY_C_CONTIGUOUS) else: clear_flags(arr, NPY.ARRAY_C_CONTIGUOUS) - sd = arr.dtype.elsize - for i in range(len(shape)): - dim = shape[i] - if strides[i] != sd: - clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS) - return - if dim == 0: - break - sd *= dim - enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS) + is_f_contig = is_f_contiguous(arr) + if is_f_contig: + enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS) + else: + clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS) class W_FlagsObject(W_Root): diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py --- a/pypy/module/micronumpy/flatiter.py +++ b/pypy/module/micronumpy/flatiter.py @@ -22,6 +22,9 @@ def get_shape(self): return self.shape + def get_size(self): + return self.base().get_size() + def create_iter(self, shape=None, backward_broadcast=False): assert isinstance(self.base(), W_NDimArray) return self.base().create_iter() @@ -41,8 +44,8 @@ return space.wrap(self.state.index) def descr_coords(self, space): - self.state = self.iter.update(self.state) - return space.newtuple([space.wrap(c) for c in self.state.indices]) + coords = self.iter.indices(self.state) + return space.newtuple([space.wrap(c) for c in coords]) def descr_iter(self): return self @@ -54,7 +57,7 @@ if self.iter.done(self.state): raise OperationError(space.w_StopIteration, space.w_None) w_res = self.iter.getitem(self.state) - self.state = self.iter.next(self.state) + self.iter.next(self.state, mutate=True) return w_res def descr_getitem(self, space, w_idx): @@ -71,7 +74,7 @@ base.get_order(), w_instance=base) return loop.flatiter_getitem(res, self.iter, state, step) finally: - self.state = self.iter.reset(self.state) + self.iter.reset(self.state, mutate=True) def descr_setitem(self, space, w_idx, w_value): if not (space.isinstance_w(w_idx, space.w_int) or @@ -91,7 +94,7 @@ arr = convert_to_array(space, w_value) loop.flatiter_setitem(space, dtype, arr, self.iter, state, step, length) finally: - self.state = self.iter.reset(self.state) + self.iter.reset(self.state, mutate=True) W_FlatIterator.typedef = TypeDef("numpy.flatiter", diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py --- a/pypy/module/micronumpy/iterators.py +++ b/pypy/module/micronumpy/iterators.py @@ -41,16 +41,6 @@ from pypy.module.micronumpy.base import W_NDimArray from pypy.module.micronumpy.flagsobj import _update_contiguous_flags -class OpFlag(object): - def __init__(self): - self.rw = '' - self.broadcast = True - self.force_contig = False - self.force_align = False - self.native_byte_order = False - self.tmp_copy = '' - self.allocate = False - class PureShapeIter(object): def __init__(self, shape, idx_w): @@ -87,25 +77,24 @@ class IterState(object): - _immutable_fields_ = ['iterator', 'index', 'indices', 'offset'] + _immutable_fields_ = ['iterator', '_indices'] def __init__(self, iterator, index, indices, offset): self.iterator = iterator self.index = index - self.indices = indices + self._indices = indices self.offset = offset class ArrayIter(object): _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]', 'strides[*]', 'backstrides[*]', 'factors[*]', - 'slice_shape', 'slice_stride', 'slice_backstride', - 'track_index', 'operand_type', 'slice_operand_type'] + 'track_index'] track_index = True - def __init__(self, array, size, shape, strides, backstrides, op_flags=OpFlag()): - from pypy.module.micronumpy import concrete + @jit.unroll_safe + def __init__(self, array, size, shape, strides, backstrides): assert len(shape) == len(strides) == len(backstrides) _update_contiguous_flags(array) self.contiguous = (array.flags & NPY.ARRAY_C_CONTIGUOUS and @@ -117,12 +106,6 @@ self.shape_m1 = [s - 1 for s in shape] self.strides = strides self.backstrides = backstrides - self.slice_shape = 1 - self.slice_stride = -1 - if strides: - self.slice_stride = strides[-1] - self.slice_backstride = 1 - self.slice_operand_type = concrete.SliceArray ndim = len(shape) factors = [0] * ndim @@ -132,32 +115,35 @@ else: factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i] self.factors = factors - if op_flags.rw == 'r': - self.operand_type = concrete.ConcreteNonWritableArrayWithBase - else: - self.operand_type = concrete.ConcreteArrayWithBase @jit.unroll_safe - def reset(self, state=None): + def reset(self, state=None, mutate=False): + index = 0 if state is None: indices = [0] * len(self.shape_m1) else: assert state.iterator is self - indices = state.indices + indices = state._indices for i in xrange(self.ndim_m1, -1, -1): indices[i] = 0 - return IterState(self, 0, indices, self.array.start) + offset = self.array.start + if not mutate: + return IterState(self, index, indices, offset) + state.index = index + state.offset = offset @jit.unroll_safe - def next(self, state): + def next(self, state, mutate=False): assert state.iterator is self index = state.index if self.track_index: index += 1 - indices = state.indices[:] + indices = state._indices offset = state.offset if self.contiguous: offset += self.array.dtype.elsize + elif self.ndim_m1 == 0: + offset += self.strides[0] else: for i in xrange(self.ndim_m1, -1, -1): idx = indices[i] @@ -168,13 +154,18 @@ else: indices[i] = 0 offset -= self.backstrides[i] - return IterState(self, index, indices, offset) + if not mutate: + return IterState(self, index, indices, offset) + state.index = index + state.offset = offset @jit.unroll_safe def goto(self, index): offset = self.array.start if self.contiguous: offset += index * self.array.dtype.elsize + elif self.ndim_m1 == 0: + offset += index * self.strides[0] else: current = index for i in xrange(len(self.shape_m1)): @@ -183,20 +174,20 @@ return IterState(self, index, None, offset) @jit.unroll_safe - def update(self, state): + def indices(self, state): assert state.iterator is self assert self.track_index - if not self.contiguous: - return state + indices = state._indices + if not (self.contiguous or self.ndim_m1 == 0): + return indices current = state.index - indices = state.indices for i in xrange(len(self.shape_m1)): if self.factors[i] != 0: indices[i] = current / self.factors[i] current %= self.factors[i] else: indices[i] = 0 - return IterState(self, state.index, indices, state.offset) + return indices def done(self, state): assert state.iterator is self @@ -215,12 +206,6 @@ assert state.iterator is self self.array.setitem(state.offset, elem) - def getoperand(self, st, base): - impl = self.operand_type - res = impl([], self.array.dtype, self.array.order, [], [], - self.array.storage, base) - res.start = st.offset - return res def AxisIter(array, shape, axis, cumulative): strides = array.get_strides() @@ -244,42 +229,3 @@ size /= shape[axis] shape[axis] = backstrides[axis] = 0 return ArrayIter(array, size, shape, array.strides, backstrides) - -class SliceIter(ArrayIter): - ''' - used with external loops, getitem and setitem return a SliceArray - view into the original array - ''' - _immutable_fields_ = ['base', 'slice_shape[*]', 'slice_stride[*]', 'slice_backstride[*]'] - - def __init__(self, array, size, shape, strides, backstrides, slice_shape, - slice_stride, slice_backstride, op_flags, base): - from pypy.module.micronumpy import concrete - ArrayIter.__init__(self, array, size, shape, strides, backstrides, op_flags) - self.slice_shape = slice_shape - self.slice_stride = slice_stride - self.slice_backstride = slice_backstride - self.base = base - if op_flags.rw == 'r': - self.slice_operand_type = concrete.NonWritableSliceArray - else: - self.slice_operand_type = concrete.SliceArray - - def getitem(self, state): - # XXX cannot be called - must return a boxed value - assert False - - def getitem_bool(self, state): - # XXX cannot be called - must return a boxed value - assert False - - def setitem(self, state, elem): - # XXX cannot be called - must return a boxed value - assert False - - def getoperand(self, state, base): - assert state.iterator is self - impl = self.slice_operand_type - arr = impl(state.offset, [self.slice_stride], [self.slice_backstride], - [self.slice_shape], self.array, self.base) - return arr diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -42,23 +42,38 @@ # TODO handle __array_priorities__ and maybe flip the order + if w_lhs.get_size() == 1: + w_left = w_lhs.get_scalar_value().convert_to(space, calc_dtype) + left_iter = left_state = None + else: + w_left = None + left_iter, left_state = w_lhs.create_iter(shape) + left_iter.track_index = False + + if w_rhs.get_size() == 1: + w_right = w_rhs.get_scalar_value().convert_to(space, calc_dtype) + right_iter = right_state = None + else: + w_right = None + right_iter, right_state = w_rhs.create_iter(shape) + right_iter.track_index = False + if out is None: out = W_NDimArray.from_shape(space, shape, res_dtype, w_instance=lhs_for_subtype) - left_iter, left_state = w_lhs.create_iter(shape) - right_iter, right_state = w_rhs.create_iter(shape) out_iter, out_state = out.create_iter(shape) - left_iter.track_index = right_iter.track_index = False shapelen = len(shape) while not out_iter.done(out_state): call2_driver.jit_merge_point(shapelen=shapelen, func=func, calc_dtype=calc_dtype, res_dtype=res_dtype) - w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype) - w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype) + if left_iter: + w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype) + left_state = left_iter.next(left_state) + if right_iter: + w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype) + right_state = right_iter.next(right_state) out_iter.setitem(out_state, func(calc_dtype, w_left, w_right).convert_to( space, res_dtype)) - left_state = left_iter.next(left_state) - right_state = right_iter.next(right_state) out_state = out_iter.next(out_state) return out @@ -68,11 +83,12 @@ reds='auto') def call1(space, shape, func, calc_dtype, res_dtype, w_obj, out): + obj_iter, obj_state = w_obj.create_iter(shape) + obj_iter.track_index = False + if out is None: out = W_NDimArray.from_shape(space, shape, res_dtype, w_instance=w_obj) - obj_iter, obj_state = w_obj.create_iter(shape) out_iter, out_state = out.create_iter(shape) - obj_iter.track_index = False shapelen = len(shape) while not out_iter.done(out_state): call1_driver.jit_merge_point(shapelen=shapelen, func=func, @@ -89,17 +105,14 @@ def setslice(space, shape, target, source): if not shape: - # XXX - simplify - target_iter, target_state = target.create_iter(shape) - source_iter, source_state = source.create_iter(shape) dtype = target.dtype - val = source_iter.getitem(source_state) + val = source.getitem(source.start) if dtype.is_str_or_unicode(): val = dtype.coerce(space, val) else: val = val.convert_to(space, dtype) - target_iter.setitem(target_state, val) - return target + target.setitem(target.start, val) + return target return _setslice(space, shape, target, source) def _setslice(space, shape, target, source): @@ -107,6 +120,7 @@ # array implementations, not arrays target_iter, target_state = target.create_iter(shape) source_iter, source_state = source.create_iter(shape) + source_iter.track_index = False dtype = target.dtype shapelen = len(shape) while not target_iter.done(target_state): @@ -152,6 +166,7 @@ def compute_reduce_cumulative(space, obj, out, calc_dtype, func, identity): obj_iter, obj_state = obj.create_iter() out_iter, out_state = out.create_iter() + out_iter.track_index = False if identity is None: cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) out_iter.setitem(out_state, cur_value) @@ -225,10 +240,9 @@ state = x_state return out -axis_reduce__driver = jit.JitDriver(name='numpy_axis_reduce', - greens=['shapelen', - 'func', 'dtype'], - reds='auto') +axis_reduce_driver = jit.JitDriver(name='numpy_axis_reduce', + greens=['shapelen', 'func', 'dtype'], + reds='auto') def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity, cumulative, temp): @@ -241,21 +255,24 @@ temp_iter = out_iter # hack temp_state = out_state arr_iter, arr_state = arr.create_iter() + arr_iter.track_index = False if identity is not None: identity = identity.convert_to(space, dtype) shapelen = len(shape) while not out_iter.done(out_state): - axis_reduce__driver.jit_merge_point(shapelen=shapelen, func=func, - dtype=dtype) - assert not arr_iter.done(arr_state) + axis_reduce_driver.jit_merge_point(shapelen=shapelen, func=func, + dtype=dtype) w_val = arr_iter.getitem(arr_state).convert_to(space, dtype) - out_state = out_iter.update(out_state) - if out_state.indices[axis] == 0: + arr_state = arr_iter.next(arr_state) + + out_indices = out_iter.indices(out_state) + if out_indices[axis] == 0: if identity is not None: w_val = func(dtype, identity, w_val) else: cur = temp_iter.getitem(temp_state) w_val = func(dtype, cur, w_val) + out_iter.setitem(out_state, w_val) out_state = out_iter.next(out_state) if cumulative: @@ -263,7 +280,6 @@ temp_state = temp_iter.next(temp_state) else: temp_state = out_state - arr_state = arr_iter.next(arr_state) return out @@ -382,9 +398,9 @@ while not arr_iter.done(arr_state): nonzero_driver.jit_merge_point(shapelen=shapelen, dims=dims, dtype=dtype) if arr_iter.getitem_bool(arr_state): - arr_state = arr_iter.update(arr_state) + arr_indices = arr_iter.indices(arr_state) for d in dims: - res_iter.setitem(res_state, box(arr_state.indices[d])) + res_iter.setitem(res_state, box(arr_indices[d])) res_state = res_iter.next(res_state) arr_state = arr_iter.next(arr_state) return res diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -19,7 +19,7 @@ order_converter, shape_converter, searchside_converter from pypy.module.micronumpy.flagsobj import W_FlagsObject from pypy.module.micronumpy.strides import get_shape_from_iterable, \ - shape_agreement, shape_agreement_multiple + shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous def _match_dot_shapes(space, left, right): @@ -529,9 +529,10 @@ "__array__(dtype) not implemented")) if type(self) is W_NDimArray: return self + sz = support.product(self.get_shape()) * self.get_dtype().elsize return W_NDimArray.from_shape_and_storage( space, self.get_shape(), self.implementation.storage, - self.get_dtype(), w_base=self) + self.get_dtype(), storage_bytes=sz, w_base=self) def descr_array_iface(self, space): addr = self.implementation.get_storage_as_int(space) @@ -827,7 +828,15 @@ raise OperationError(space.w_ValueError, space.wrap( "new type not compatible with array.")) else: - if dims == 1 or impl.get_strides()[0] < impl.get_strides()[-1]: + if not is_c_contiguous(impl) and not is_f_contiguous(impl): + if old_itemsize != new_itemsize: + raise OperationError(space.w_ValueError, space.wrap( + "new type not compatible with array.")) + # Strides, shape does not change + v = impl.astype(space, dtype) + return wrap_impl(space, w_type, self, v) + strides = impl.get_strides() + if dims == 1 or strides[0] <strides[-1]: # Column-major, resize first dimension if new_shape[0] * old_itemsize % new_itemsize != 0: raise OperationError(space.w_ValueError, space.wrap( @@ -1119,9 +1128,9 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.module.micronumpy.concrete import SliceArray - numpypy = space.getbuiltinmodule("_numpypy") - assert isinstance(numpypy, MixedModule) - multiarray = numpypy.get("multiarray") + _numpypy = space.getbuiltinmodule("_numpypy") + assert isinstance(_numpypy, MixedModule) + multiarray = _numpypy.get("multiarray") assert isinstance(multiarray, MixedModule) reconstruct = multiarray.get("_reconstruct") parameters = space.newtuple([self.getclass(space), space.newtuple( @@ -1169,8 +1178,8 @@ "improper dtype '%R'", dtype) self.implementation = W_NDimArray.from_shape_and_storage( space, [space.int_w(i) for i in space.listview(shape)], - rffi.str2charp(space.str_w(storage), track_allocation=False), - dtype, owning=True).implementation + rffi.str2charp(space.str_w(storage), track_allocation=False), + dtype, storage_bytes=space.len_w(storage), owning=True).implementation def descr___array_finalize__(self, space, w_obj): pass @@ -1194,8 +1203,10 @@ if not space.is_none(w_buffer): if (not space.is_none(w_strides)): - raise OperationError(space.w_NotImplementedError, - space.wrap("unsupported param")) + strides = [space.int_w(w_i) for w_i in + space.unpackiterable(w_strides)] + else: + strides = None try: buf = space.writebuf_w(w_buffer) @@ -1209,16 +1220,14 @@ if not shape: raise OperationError(space.w_TypeError, space.wrap( "numpy scalars from buffers not supported yet")) - totalsize = support.product(shape) * dtype.elsize - if totalsize + offset > buf.getlength(): - raise OperationError(space.w_TypeError, space.wrap( - "buffer is too small for requested array")) storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr) storage = rffi.ptradd(storage, offset) - return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype, + return W_NDimArray.from_shape_and_storage(space, shape, storage, + dtype, w_base=w_buffer, + storage_bytes=buf.getlength()-offset, w_subtype=w_subtype, - w_base=w_buffer, - writable=not buf.readonly) + writable=not buf.readonly, + strides=strides) order = order_converter(space, w_order, NPY.CORDER) if order == NPY.CORDER: @@ -1236,8 +1245,9 @@ return w_ret -@unwrap_spec(addr=int) -def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype, w_subtype=None): +@unwrap_spec(addr=int, buf_len=int) +def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype, + buf_len=-1, w_subtype=None, w_strides=None): """ Create an array from an existing buffer, given its address as int. PyPy-only implementation detail. @@ -1246,14 +1256,22 @@ dtype = space.interp_w(descriptor.W_Dtype, space.call_function( space.gettypefor(descriptor.W_Dtype), w_dtype)) shape = shape_converter(space, w_shape, dtype) + if not space.is_none(w_strides): + strides = [space.int_w(w_i) for w_i in + space.unpackiterable(w_strides)] + else: + strides = None if w_subtype: if not space.isinstance_w(w_subtype, space.w_type): raise OperationError(space.w_ValueError, space.wrap( "subtype must be a subtype of ndarray, not a class instance")) return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype, - 'C', False, w_subtype) + buf_len, 'C', False, w_subtype, + strides=strides) else: - return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype) + return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype, + storage_bytes=buf_len, + strides=strides) app_take = applevel(r""" def take(a, indices, axis, out, mode): diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py --- a/pypy/module/micronumpy/nditer.py +++ b/pypy/module/micronumpy/nditer.py @@ -6,7 +6,7 @@ from pypy.module.micronumpy import ufuncs, support, concrete from pypy.module.micronumpy.base import W_NDimArray, convert_to_array from pypy.module.micronumpy.descriptor import decode_w_dtype -from pypy.module.micronumpy.iterators import ArrayIter, SliceIter, OpFlag +from pypy.module.micronumpy.iterators import ArrayIter from pypy.module.micronumpy.strides import (calculate_broadcast_strides, shape_agreement, shape_agreement_multiple) @@ -36,6 +36,16 @@ return ret +class OpFlag(object): + def __init__(self): + self.rw = '' + self.broadcast = True + self.force_contig = False + self.force_align = False + self.native_byte_order = False + self.tmp_copy = '' + self.allocate = False + def parse_op_flag(space, lst): op_flag = OpFlag() for w_item in lst: @@ -142,11 +152,73 @@ raise NotImplementedError('not implemented yet') -def get_iter(space, order, arr, shape, dtype, op_flags): +class OperandIter(ArrayIter): + _immutable_fields_ = ['slice_shape', 'slice_stride', 'slice_backstride', + 'operand_type', 'base'] + + def getitem(self, state): + # cannot be called - must return a boxed value + assert False + + def getitem_bool(self, state): + # cannot be called - must return a boxed value + assert False + + def setitem(self, state, elem): + # cannot be called - must return a boxed value + assert False + + +class ConcreteIter(OperandIter): + def __init__(self, array, size, shape, strides, backstrides, + op_flags, base): + OperandIter.__init__(self, array, size, shape, strides, backstrides) + self.slice_shape = 1 + self.slice_stride = -1 + if strides: + self.slice_stride = strides[-1] + self.slice_backstride = 1 + if op_flags.rw == 'r': + self.operand_type = concrete.ConcreteNonWritableArrayWithBase + else: + self.operand_type = concrete.ConcreteArrayWithBase + self.base = base + + def getoperand(self, state): + assert state.iterator is self + impl = self.operand_type + res = impl([], self.array.dtype, self.array.order, [], [], + self.array.storage, self.base) + res.start = state.offset + return res + + +class SliceIter(OperandIter): + def __init__(self, array, size, shape, strides, backstrides, slice_shape, + slice_stride, slice_backstride, op_flags, base): + OperandIter.__init__(self, array, size, shape, strides, backstrides) + self.slice_shape = slice_shape + self.slice_stride = slice_stride + self.slice_backstride = slice_backstride + if op_flags.rw == 'r': + self.operand_type = concrete.NonWritableSliceArray + else: + self.operand_type = concrete.SliceArray + self.base = base + + def getoperand(self, state): + assert state.iterator is self + impl = self.operand_type + arr = impl(state.offset, [self.slice_stride], [self.slice_backstride], + [self.slice_shape], self.array, self.base) + return arr + + +def get_iter(space, order, arr, shape, dtype, op_flags, base): imp = arr.implementation backward = is_backward(imp, order) if arr.is_scalar(): - return ArrayIter(imp, 1, [], [], [], op_flags=op_flags) + return ConcreteIter(imp, 1, [], [], [], op_flags, base) if (imp.strides[0] < imp.strides[-1] and not backward) or \ (imp.strides[0] > imp.strides[-1] and backward): # flip the strides. Is this always true for multidimension? @@ -161,7 +233,7 @@ backstrides = imp.backstrides r = calculate_broadcast_strides(strides, backstrides, imp.shape, shape, backward) - return ArrayIter(imp, imp.get_size(), shape, r[0], r[1], op_flags=op_flags) + return ConcreteIter(imp, imp.get_size(), shape, r[0], r[1], op_flags, base) def calculate_ndim(op_in, oa_ndim): if oa_ndim >=0: @@ -398,7 +470,7 @@ self.iters = [] for i in range(len(self.seq)): it = get_iter(space, self.order, self.seq[i], self.shape, - self.dtypes[i], self.op_flags[i]) + self.dtypes[i], self.op_flags[i], self) it.contiguous = False self.iters.append((it, it.reset())) @@ -437,7 +509,7 @@ return space.wrap(self) def getitem(self, it, st): - res = it.getoperand(st, self) + res = it.getoperand(st) return W_NDimArray(res) def descr_getitem(self, space, w_idx): @@ -455,6 +527,7 @@ def descr_len(self, space): space.wrap(len(self.iters)) + @jit.unroll_safe def descr_next(self, space): for it, st in self.iters: if not it.done(st): diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -270,7 +270,7 @@ shape = shape_agreement(space, shape, arr) return shape - +@jit.unroll_safe def _shape_agreement(shape1, shape2): """ Checks agreement about two shapes with respect to broadcasting. Returns the resulting shape. @@ -362,6 +362,13 @@ backstrides.reverse() return strides, backstrides +@jit.unroll_safe +def calc_backstrides(strides, shape): + ndims = len(shape) + new_backstrides = [0] * ndims + for nd in range(ndims): + new_backstrides[nd] = (shape[nd] - 1) * strides[nd] + return new_backstrides # Recalculating strides. Find the steps that the iteration does for each # dimension, given the stride and shape. Then try to create a new stride that @@ -422,3 +429,35 @@ n_old_elems_to_use *= old_shape[oldI] assert len(new_strides) == len(new_shape) return new_strides[:] + +@jit.unroll_safe +def is_c_contiguous(arr): + shape = arr.get_shape() + strides = arr.get_strides() + ret = True + sd = arr.dtype.elsize + for i in range(len(shape) - 1, -1, -1): + dim = shape[i] + if strides[i] != sd: + ret = False + break + if dim == 0: + break + sd *= dim + return ret + +@jit.unroll_safe +def is_f_contiguous(arr): + shape = arr.get_shape() + strides = arr.get_strides() + ret = True + sd = arr.dtype.elsize + for i in range(len(shape)): + dim = shape[i] + if strides[i] != sd: + ret = False + break + if dim == 0: + break + sd *= dim + return ret diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py --- a/pypy/module/micronumpy/test/test_arrayops.py +++ b/pypy/module/micronumpy/test/test_arrayops.py @@ -3,13 +3,13 @@ class AppTestNumSupport(BaseNumpyAppTest): def test_zeros(self): - from numpypy import zeros + from numpy import zeros a = zeros(3) assert len(a) == 3 assert a[0] == a[1] == a[2] == 0 def test_empty(self): - from numpypy import empty + from numpy import empty import gc for i in range(1000): a = empty(3) @@ -26,26 +26,26 @@ "empty() returned a zeroed out array every time") def test_where(self): - from numpypy import where, ones, zeros, array + from numpy import where, ones, zeros, array a = [1, 2, 3, 0, -3] a = where(array(a) > 0, ones(5), zeros(5)) assert (a == [1, 1, 1, 0, 0]).all() def test_where_differing_dtypes(self): - from numpypy import array, ones, zeros, where + from numpy import array, ones, zeros, where a = [1, 2, 3, 0, -3] _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit