[pypy-commit] pypy default: add shortcut to ensure that 'for c in uni' does not compute the index storage

[cfbolz]

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r96035:637f18678c1c
Date: 2019-02-17 12:32 +0100
http://bitbucket.org/pypy/pypy/changeset/637f18678c1c/

Log:add shortcut to ensure that 'for c in uni' does not compute the
index storage

diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -92,6 +92,33 @@
 return w_item
 
 
+class W_FastUnicodeIterObject(W_AbstractSeqIterObject):
+"""Sequence iterator specialized for unicode objects."""
+
+def __init__(self, w_seq):
+from pypy.objspace.std.unicodeobject import W_UnicodeObject
+W_AbstractSeqIterObject.__init__(self, w_seq)
+assert isinstance(w_seq, W_UnicodeObject)
+self.byteindex = 0
+
+def descr_next(self, space):
+from pypy.objspace.std.unicodeobject import W_UnicodeObject
+from rpython.rlib import rutf8
+w_seq = self.w_seq
+if w_seq is None:
+raise OperationError(space.w_StopIteration, space.w_None)
+assert isinstance(w_seq, W_UnicodeObject)
+index = self.index
+if index == w_seq._length:
+self.w_seq = None
+raise OperationError(space.w_StopIteration, space.w_None)
+start = self.byteindex
+end = rutf8.next_codepoint_pos(w_seq._utf8, start)
+w_res = W_UnicodeObject(w_seq._utf8[start:end], 1)
+self.byteindex = end
+return w_res
+
+
 class W_FastTupleIterObject(W_AbstractSeqIterObject):
 """Sequence iterator specialized for tuples, accessing directly
 their RPython-level list of wrapped objects.
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -22,6 +22,7 @@
 from pypy.objspace.std.floatobject import W_FloatObject
 from pypy.objspace.std.intobject import W_IntObject, setup_prebuilt, wrapint
 from pypy.objspace.std.iterobject import W_AbstractSeqIterObject, 
W_SeqIterObject
+from pypy.objspace.std.iterobject import W_FastUnicodeIterObject
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.objspace.std.longobject import W_LongObject, newlong
 from pypy.objspace.std.memoryobject import W_MemoryView
@@ -339,6 +340,8 @@
 return W_SliceObject(w_start, w_end, w_step)
 
 def newseqiter(self, w_obj):
+if type(w_obj) is W_UnicodeObject:
+return W_FastUnicodeIterObject(w_obj)
 return W_SeqIterObject(w_obj)
 
 def newbuffer(self, obj):
diff --git a/pypy/objspace/std/test/test_unicodeobject.py 
b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -41,6 +41,18 @@
 space.w_unicode, "__new__", space.w_unicode, w_uni)
 assert w_new is w_uni
 
+def test_fast_iter(self):
+space = self.space
+w_uni = space.newutf8(u"aä".encode("utf-8"), 2)
+old_index_storage = w_uni._index_storage
+w_iter = space.iter(w_uni)
+w_char1 = w_iter.descr_next(space)
+w_char2 = w_iter.descr_next(space)
+assert w_uni._index_storage is old_index_storage
+assert space.eq_w(w_char1, w_uni._getitem_result(space, 0))
+assert space.eq_w(w_char2, w_uni._getitem_result(space, 1))
+
+
 if HAS_HYPOTHESIS:
 @given(strategies.text(), strategies.integers(min_value=0, 
max_value=10),
   strategies.integers(min_value=-1, 
max_value=10))
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.5: merge default

[cfbolz]

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.5
Changeset: r96036:c87433506699
Date: 2019-02-17 12:35 +0100
http://bitbucket.org/pypy/pypy/changeset/c87433506699/

Log:merge default

diff --git a/extra_tests/cffi_tests/cffi0/test_ownlib.py 
b/extra_tests/cffi_tests/cffi0/test_ownlib.py
--- a/extra_tests/cffi_tests/cffi0/test_ownlib.py
+++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py
@@ -352,6 +352,8 @@
 def test_modify_struct_value(self):
 if self.module is None:
 py.test.skip("fix the auto-generation of the tiny test lib")
+if self.Backend is CTypesBackend:
+py.test.skip("fails with the ctypes backend on some architectures")
 ffi = FFI(backend=self.Backend())
 ffi.cdef("""
 typedef struct {
diff --git a/extra_tests/cffi_tests/embedding/thread3-test.c 
b/extra_tests/cffi_tests/embedding/thread3-test.c
--- a/extra_tests/cffi_tests/embedding/thread3-test.c
+++ b/extra_tests/cffi_tests/embedding/thread3-test.c
@@ -52,5 +52,6 @@
 assert(status == 0);
 }
 printf("done\n");
+fflush(stdout);   /* this is occasionally needed on Windows */
 return 0;
 }
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -5,8 +5,8 @@
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 from .error import PkgConfigError
 
-__version__ = "1.12.0"
-__version_info__ = (1, 12, 0)
+__version__ = "1.12.1"
+__version_info__ = (1, 12, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,43 +8,20 @@
the same works for the other two macros.  Py_DEBUG implies them,
but not the other way around.
 
-   The implementation is messy (issue #350): on Windows, with _MSC_VER,
-   we have to define Py_LIMITED_API even before including pyconfig.h.
-   In that case, we guess what pyconfig.h will do to the macros above,
-   and check our guess after the #include.
-
-   Note that on Windows, with CPython 3.x, you need virtualenv version
-   >= 16.0.0.  Older versions don't copy PYTHON3.DLL.  As a workaround
-   you can remove the definition of Py_LIMITED_API here.
-
-   See also 'py_limited_api' in cffi/setuptools_ext.py.
+   Issue #350 is still open: on Windows, the code here causes it to link
+   with PYTHON36.DLL (for example) instead of PYTHON3.DLL.  A fix was
+   attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv
+   does not make PYTHON3.DLL available, and so the "correctly" compiled
+   version would not run inside a virtualenv.  We will re-apply the fix
+   after virtualenv has been fixed for some time.  For explanation, see
+   issue #355.  For a workaround if you want PYTHON3.DLL and don't worry
+   about virtualenv, see issue #350.  See also 'py_limited_api' in
+   setuptools_ext.py.
 */
 #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
-#  ifdef _MSC_VER
-#if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && 
!defined(Py_REF_DEBUG)
-#  define Py_LIMITED_API
-#endif
-#include 
- /* sanity-check: Py_LIMITED_API will cause crashes if any of these
-are also defined.  Normally, the Python file PC/pyconfig.h does not
-cause any of these to be defined, with the exception that _DEBUG
-causes Py_DEBUG.  Double-check that. */
-#ifdef Py_LIMITED_API
-#  if defined(Py_DEBUG)
-#error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API 
is set"
-#  endif
-#  if defined(Py_TRACE_REFS)
-#error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but 
Py_LIMITED_API is set"
-#  endif
-#  if defined(Py_REF_DEBUG)
-#error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but 
Py_LIMITED_API is set"
-#  endif
-#endif
-#  else
-#include 
-#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-#  define Py_LIMITED_API
-#endif
+#  include 
+#  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+#define Py_LIMITED_API
 #  endif
 #endif
 
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -221,7 +221,7 @@
 
 if (f != NULL && f != Py_None) {
 PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-   "\ncompiled with cffi version: 1.12.0"
+   "\ncompiled with cffi version: 1.12.1"
"\n_cffi_backend module: ", f);
 modules = PyImport_GetModuleDict();
 mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py

[pypy-commit] pypy py3.5: doesn't need the hack on py3.5

[cfbolz]

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.5
Changeset: r96039:227b54cdf79b
Date: 2019-02-17 13:35 +0100
http://bitbucket.org/pypy/pypy/changeset/227b54cdf79b/

Log:doesn't need the hack on py3.5

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -366,8 +366,6 @@
 return W_SliceObject(w_start, w_end, w_step)
 
 def newseqiter(self, w_obj):
-if type(w_obj) is W_UnicodeObject:
-return W_FastUnicodeIterObject(w_obj)
 return W_SeqIterObject(w_obj)
 
 def newmemoryview(self, w_obj):
diff --git a/pypy/objspace/std/unicodeobject.py 
b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -93,6 +93,10 @@
 def listview_utf8(self):
 return _create_list_from_unicode(self._utf8)
 
+def descr_iter(self, space):
+from pypy.objspace.std.iterobject import W_FastUnicodeIterObject
+return W_FastUnicodeIterObject(self)
+
 def ord(self, space):
 if self._len() != 1:
 raise oefmt(space.w_TypeError,
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.6: merge py3.5

[cfbolz]

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.6
Changeset: r96040:d6a918e4c217
Date: 2019-02-17 13:39 +0100
http://bitbucket.org/pypy/pypy/changeset/d6a918e4c217/

Log:merge py3.5

diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -100,6 +100,34 @@
 return w_item
 
 
+class W_FastUnicodeIterObject(W_AbstractSeqIterObject):
+"""Sequence iterator specialized for unicode objects."""
+
+def __init__(self, w_seq):
+from pypy.objspace.std.unicodeobject import W_UnicodeObject
+W_AbstractSeqIterObject.__init__(self, w_seq)
+assert isinstance(w_seq, W_UnicodeObject)
+self.byteindex = 0
+
+def descr_next(self, space):
+from pypy.objspace.std.unicodeobject import W_UnicodeObject
+from rpython.rlib import rutf8
+w_seq = self.w_seq
+if w_seq is None:
+raise OperationError(space.w_StopIteration, space.w_None)
+assert isinstance(w_seq, W_UnicodeObject)
+index = self.index
+if index == w_seq._length:
+self.w_seq = None
+raise OperationError(space.w_StopIteration, space.w_None)
+start = self.byteindex
+end = rutf8.next_codepoint_pos(w_seq._utf8, start)
+w_res = W_UnicodeObject(w_seq._utf8[start:end], 1)
+self.byteindex = end
+self.index += 1
+return w_res
+
+
 class W_FastTupleIterObject(W_AbstractSeqIterObject):
 """Sequence iterator specialized for tuples, accessing directly
 their RPython-level list of wrapped objects.
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -26,6 +26,7 @@
 from pypy.objspace.std.intobject import (
 W_AbstractIntObject, W_IntObject, setup_prebuilt, wrapint)
 from pypy.objspace.std.iterobject import W_AbstractSeqIterObject, 
W_SeqIterObject
+from pypy.objspace.std.iterobject import W_FastUnicodeIterObject
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.objspace.std.longobject import W_LongObject, newlong
 from pypy.objspace.std.memoryobject import W_MemoryView
diff --git a/pypy/objspace/std/test/test_unicodeobject.py 
b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -38,6 +38,19 @@
 space.w_unicode, "__new__", space.w_unicode, w_uni)
 assert w_new is w_uni
 
+def test_fast_iter(self):
+space = self.space
+w_uni = space.newutf8(u"aä".encode("utf-8"), 2)
+old_index_storage = w_uni._index_storage
+w_iter = space.iter(w_uni)
+w_char1 = w_iter.descr_next(space)
+w_char2 = w_iter.descr_next(space)
+py.test.raises(OperationError, w_iter.descr_next, space)
+assert w_uni._index_storage is old_index_storage
+assert space.eq_w(w_char1, w_uni._getitem_result(space, 0))
+assert space.eq_w(w_char2, w_uni._getitem_result(space, 1))
+
+
 if HAS_HYPOTHESIS:
 @given(strategies.text(), strategies.integers(min_value=0, 
max_value=10),
   strategies.integers(min_value=-1, 
max_value=10))
diff --git a/pypy/objspace/std/unicodeobject.py 
b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -93,6 +93,10 @@
 def listview_utf8(self):
 return _create_list_from_unicode(self._utf8)
 
+def descr_iter(self, space):
+from pypy.objspace.std.iterobject import W_FastUnicodeIterObject
+return W_FastUnicodeIterObject(self)
+
 def ord(self, space):
 if self._len() != 1:
 raise oefmt(space.w_TypeError,
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.5: merge default

[cfbolz]

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.5
Changeset: r96038:1c54055fec69
Date: 2019-02-17 12:53 +0100
http://bitbucket.org/pypy/pypy/changeset/1c54055fec69/

Log:merge default

diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -124,6 +124,7 @@
 end = rutf8.next_codepoint_pos(w_seq._utf8, start)
 w_res = W_UnicodeObject(w_seq._utf8[start:end], 1)
 self.byteindex = end
+self.index += 1
 return w_res
 
 
diff --git a/pypy/objspace/std/test/test_unicodeobject.py 
b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -45,6 +45,7 @@
 w_iter = space.iter(w_uni)
 w_char1 = w_iter.descr_next(space)
 w_char2 = w_iter.descr_next(space)
+py.test.raises(OperationError, w_iter.descr_next, space)
 assert w_uni._index_storage is old_index_storage
 assert space.eq_w(w_char1, w_uni._getitem_result(space, 0))
 assert space.eq_w(w_char2, w_uni._getitem_result(space, 1))
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: fix

[cfbolz]

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r96037:b42bf472ce7d
Date: 2019-02-17 12:52 +0100
http://bitbucket.org/pypy/pypy/changeset/b42bf472ce7d/

Log:fix

diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -116,6 +116,7 @@
 end = rutf8.next_codepoint_pos(w_seq._utf8, start)
 w_res = W_UnicodeObject(w_seq._utf8[start:end], 1)
 self.byteindex = end
+self.index += 1
 return w_res
 
 
diff --git a/pypy/objspace/std/test/test_unicodeobject.py 
b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -48,6 +48,7 @@
 w_iter = space.iter(w_uni)
 w_char1 = w_iter.descr_next(space)
 w_char2 = w_iter.descr_next(space)
+py.test.raises(OperationError, w_iter.descr_next, space)
 assert w_uni._index_storage is old_index_storage
 assert space.eq_w(w_char1, w_uni._getitem_result(space, 0))
 assert space.eq_w(w_char2, w_uni._getitem_result(space, 1))
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: remove dead code

[mattip]

Author: Matti Picus 
Branch: 
Changeset: r96041:4e661aec658f
Date: 2019-02-17 16:41 +0200
http://bitbucket.org/pypy/pypy/changeset/4e661aec658f/

Log:remove dead code

diff --git a/pypy/interpreter/unicodehelper.py 
b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -918,55 +918,6 @@
 
 return result.build()
 
-@specialize.memo()
-def _encode_unicode_error_handler(space):
-# Fast version of the "strict" errors handler.
-from rpython.rlib import runicode
-def raise_unicode_exception_encode(errors, encoding, msg, uni,
-   startingpos, endingpos):
-assert isinstance(uni, unicode)
-u_len = len(uni)
-utf8 = runicode.unicode_encode_utf8sp(uni, u_len)
-raise OperationError(space.w_UnicodeEncodeError,
- space.newtuple([space.newtext(encoding),
- space.newtext(utf8, u_len),
- space.newint(startingpos),
- space.newint(endingpos),
- space.newtext(msg)]))
-return u'', None, 0
-return raise_unicode_exception_encode
-
-
-def encode_utf8(space, uni, allow_surrogates=False):
-# Note that Python3 tends to forbid *all* surrogates in utf-8.
-# If allow_surrogates=True, then revert to the Python 2 behavior
-# which never raises UnicodeEncodeError.  Surrogate pairs are then
-# allowed, either paired or lone.  A paired surrogate is considered
-# like the non-BMP character it stands for.  See also *_utf8sp().
-from rpython.rlib import runicode
-assert isinstance(uni, unicode)
-return runicode.unicode_encode_utf_8(
-uni, len(uni), "strict",
-errorhandler=_encode_unicode_error_handler(space),
-allow_surrogates=allow_surrogates)
-
-def encode_utf8sp(space, uni, allow_surrogates=True):
-xxx
-# Surrogate-preserving utf-8 encoding.  Any surrogate character
-# turns into its 3-bytes encoding, whether it is paired or not.
-# This should always be reversible, and the reverse is
-# decode_utf8sp().
-from rpython.rlib import runicode
-return runicode.unicode_encode_utf8sp(uni, len(uni))
-
-def decode_utf8sp(space, string):
-# Surrogate-preserving utf-8 decoding.  Assuming there is no
-# encoding error, it should always be reversible, and the reverse is
-# encode_utf8sp().
-return str_decode_utf8(string, "string", True, decode_never_raise,
-   allow_surrogates=True)
-
-
 # 
 # utf-16
 
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: revert part of 4e661aec658f to keep synced with py3.6

[mattip]

Author: Matti Picus 
Branch: 
Changeset: r96042:d13ae7d283ae
Date: 2019-02-17 16:53 +0200
http://bitbucket.org/pypy/pypy/changeset/d13ae7d283ae/

Log:revert part of 4e661aec658f to keep synced with py3.6

diff --git a/pypy/interpreter/unicodehelper.py 
b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -21,7 +21,7 @@
  space.newtext(msg)]))
 return raise_unicode_exception_decode
 
-def decode_never_raise(errors, encoding, msg, s, startingpos, endingpos):
+def _decode_never_raise(errors, encoding, msg, s, startingpos, endingpos):
 assert startingpos >= 0
 ux = ['\ux' + hex(ord(x))[2:].upper() for x in s[startingpos:endingpos]]
 return ''.join(ux), endingpos, 'b'
@@ -918,6 +918,13 @@
 
 return result.build()
 
+def decode_utf8sp(space, string):
+# Surrogate-preserving utf-8 decoding.  Assuming there is no
+# encoding error, it should always be reversible, and the reverse is
+# encode_utf8sp().
+return str_decode_utf8(string, "string", True, _decode_never_raise,
+   allow_surrogates=True)
+
 # 
 # utf-16
 
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit