[pypy-commit] pypy gc-hooks: hg merge default

[antocuni]

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94350:8711cbc8d281
Date: 2018-04-16 12:36 +
http://bitbucket.org/pypy/pypy/changeset/8711cbc8d281/

Log:hg merge default

diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -40,6 +40,8 @@
   sure things are ported back to the trunk and to the branch as
   necessary.
 
+* Make sure the RPython builds on the buildbot pass with no failures
+
 * Maybe bump the SOABI number in module/imp/importing. This has many
   implications, so make sure the PyPy community agrees to the change.
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,16 +3,10 @@
 ==
 
 .. this is a revision shortly after release-pypy-6.0.0
-.. startrev: 2e04adf1b89f
+.. startrev: f22145c34985
 
-.. branch: cpyext-subclass-setattr
 
-Fix for python-level classes that inherit from C-API types, previously the
-`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj`
-which led to cases where instance attributes were lost. Fixes issue #2793
+.. branch: issue2752
 
-
-.. branch: pyparser-improvements-2
-
-Improve line offsets that are reported by SyntaxError. Improve error messages
-for a few situations, including mismatched parenthesis.
+Fix a rare GC bug that was introduced more than one year ago, but was
+not diagnosed before issue #2752.
diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst 
b/pypy/doc/whatsnew-pypy2-6.0.0.rst
--- a/pypy/doc/whatsnew-pypy2-6.0.0.rst
+++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst
@@ -97,3 +97,15 @@
 Work around possible bugs in upstream ioctl users, like CPython allocate at
 least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes
 issue #2776
+
+.. branch: cpyext-subclass-setattr
+
+Fix for python-level classes that inherit from C-API types, previously the
+`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj`
+which led to cases where instance attributes were lost. Fixes issue #2793
+
+
+.. branch: pyparser-improvements-2
+
+Improve line offsets that are reported by SyntaxError. Improve error messages
+for a few situations, including mismatched parenthesis.
diff --git a/pypy/module/cpyext/test/test_bufferobject.py 
b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -4,6 +4,7 @@
 from pypy.module.cpyext.api import PyObject
 
 class AppTestBufferObject(AppTestCpythonExtensionBase):
+
 def test_FromMemory(self):
 module = self.import_extension('foo', [
 ("get_FromMemory", "METH_NOARGS",
@@ -62,3 +63,61 @@
 a = array.array('c', 'text')
 b = buffer(a)
 assert module.roundtrip(b) == 'text'
+
+
+def test_issue2752(self):
+iterations = 10
+if self.runappdirect:
+iterations = 2000
+module = self.import_extension('foo', [
+("test_mod", 'METH_VARARGS',
+"""
+PyObject *obj;
+Py_buffer bp;
+if (!PyArg_ParseTuple(args, "O", ))
+return NULL;
+
+if (PyObject_GetBuffer(obj, , PyBUF_SIMPLE) == -1)
+return NULL;
+
+if (((unsigned char*)bp.buf)[0] != '0') {
+void * buf = (void*)bp.buf;
+unsigned char val[4];
+char * s = PyString_AsString(obj);
+memcpy(val, bp.buf, 4);
+PyBuffer_Release();
+if (PyObject_GetBuffer(obj, , PyBUF_SIMPLE) == -1)
+return NULL;
+PyErr_Format(PyExc_ValueError,
+"mismatch: %p [%x %x %x %x...] now %p [%x %x %x 
%x...] as str '%s'",
+buf, val[0], val[1], val[2], val[3],
+(void *)bp.buf,
+((unsigned char*)bp.buf)[0],
+((unsigned char*)bp.buf)[1],
+((unsigned char*)bp.buf)[2],
+((unsigned char*)bp.buf)[3],
+s);
+PyBuffer_Release();
+return NULL;
+}
+
+PyBuffer_Release();
+Py_RETURN_NONE;
+"""),
+])
+bufsize = 4096
+def getdata(bufsize):
+data = b'01234567'
+for x in range(18):
+data += data
+if len(data) >= bufsize:
+break
+return data
+for j in range(iterations):
+block = getdata(bufsize)
+assert block[:8] == '01234567'
+try:
+module.test_mod(block)
+except ValueError as e:
+   

[pypy-commit] pypy gc-hooks: hg merge default

[antocuni]

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94312:f157561218b4
Date: 2018-04-12 18:03 +0200
http://bitbucket.org/pypy/pypy/changeset/f157561218b4/

Log:hg merge default

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -6,36 +6,36 @@
 Except when otherwise stated (look for LICENSE files in directories or
 information at the beginning of each file) all software and documentation in
 the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy',
-'py', and '_pytest' directories is licensed as follows: 
+'py', and '_pytest' directories is licensed as follows:
 
 The MIT License
 
-Permission is hereby granted, free of charge, to any person 
-obtaining a copy of this software and associated documentation 
-files (the "Software"), to deal in the Software without 
-restriction, including without limitation the rights to use, 
-copy, modify, merge, publish, distribute, sublicense, and/or 
-sell copies of the Software, and to permit persons to whom the 
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the
 Software is furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included 
+The above copyright notice and this permission notice shall be included
 in all copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
MERCHANTABILITY, 
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 DEALINGS IN THE SOFTWARE.
 
 
 PyPy Copyright holders 2003-2018
 
+
 
 Except when otherwise stated (look for LICENSE files or information at
 the beginning of each file) the files in the 'pypy' directory are each
-copyrighted by one or more of the following people and organizations:
+copyrighted by one or more of the following people and organizations:
 
   Armin Rigo
   Maciej Fijalkowski
@@ -89,13 +89,13 @@
   Niko Matsakis
   Alexander Hesse
   Ludovic Aubry
+  stian
   Jacob Hallen
   Jason Creighton
   Mark Young
   Alex Martelli
   Spenser Bauman
   Michal Bendowski
-  stian
   Jan de Mooij
   Tyler Wade
   Vincent Legoll
@@ -123,10 +123,10 @@
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
+  Jeremy Thurgood
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
-  Jeremy Thurgood
   Mark Pearse
   Simon Cross
   Tobias Pape
@@ -145,18 +145,19 @@
   Adrian Kuhn
   tav
   Georg Brandl
+  Joannah Nanjekye
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
-  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
+  Dodan Mihai
   Tim Felgentreff
   Eugene Oden
-  Dodan Mihai
+  Colin Valliant
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -225,12 +226,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Floris Bruynooghe
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Arianna Avanzini
   Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
@@ -250,9 +253,11 @@
   Alejandro J. Cura
   Vladimir Kryachko
   Gabriel
+  Thomas Hisch
   Mark Williams
   Kunal Grover
   Nathan Taylor
+  Barry Hart
   Travis Francis Athougies
   Yasir Suhail
   Sergey Kishchenko
@@ -260,6 +265,7 @@
   Lutz Paelike
   Ian Foote
   Philipp Rustemeuer
+  Logan Chien
   Catalin Gabriel Manciu
   Jacob Oscarson
   Ryan Gonzalez
@@ -295,7 +301,6 @@
   Akira Li
   Gustavo Niemeyer
   Rafa Gaczyski
-  Logan Chien
   Lucas Stadler
   roberto@goyle
   Matt Bogosian
@@ -308,6 +313,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Miro Hronok
   Anthony Sottile
   Nate Bragg
   Ben Darnell
@@ -315,7 +321,6 @@
   Godefroid Chappelle
   Julian Berman
   Michael Hudson-Doyle
-  Floris 

[pypy-commit] pypy gc-hooks: hg merge default

[antocuni]

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94243:00c8c55ee465
Date: 2018-04-05 09:37 +0100
http://bitbucket.org/pypy/pypy/changeset/00c8c55ee465/

Log:hg merge default

diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py
--- a/lib-python/2.7/re.py
+++ b/lib-python/2.7/re.py
@@ -225,7 +225,7 @@
 
 _pattern_type = type(sre_compile.compile("", 0))
 
-_MAXCACHE = 100
+_MAXCACHE = 1000
 
 def _compile(*key):
 # internal: compile pattern
diff --git a/lib-python/2.7/test/test_genexps.py 
b/lib-python/2.7/test/test_genexps.py
--- a/lib-python/2.7/test/test_genexps.py
+++ b/lib-python/2.7/test/test_genexps.py
@@ -87,7 +87,7 @@
 >>> dict(a = i for i in xrange(10))
 Traceback (most recent call last):
...
-SyntaxError: invalid syntax
+SyntaxError: invalid syntax (expected ')')
 
 Verify that parenthesis are required when used as a keyword argument value
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -79,3 +79,21 @@
 
 - in rare "trace is too long" situations, the JIT could break behaviour
   arbitrarily.
+
+.. branch: jit-hooks-can-be-disabled
+
+Be more efficient about JIT hooks. Make it possible for the frontend to declare
+that jit hooks are currently not enabled at all. in that case, the list of ops
+does not have to be created in the case of the on_abort hook (which is
+expensive).
+
+
+.. branch: pyparser-improvements
+
+Improve speed of Python parser, improve ParseError messages slightly.
+
+.. branch: ioctl-arg-size
+
+Work around possible bugs in upstream ioctl users, like CPython allocate at
+least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes
+issue #2776
diff --git a/pypy/interpreter/pyparser/metaparser.py 
b/pypy/interpreter/pyparser/metaparser.py
--- a/pypy/interpreter/pyparser/metaparser.py
+++ b/pypy/interpreter/pyparser/metaparser.py
@@ -147,8 +147,10 @@
 for label, next in state.arcs.iteritems():
 arcs.append((self.make_label(gram, label), 
dfa.index(next)))
 states.append((arcs, state.is_final))
-gram.dfas.append((states, self.make_first(gram, name)))
-assert len(gram.dfas) - 1 == gram.symbol_ids[name] - 256
+symbol_id = gram.symbol_ids[name]
+dfa = parser.DFA(symbol_id, states, self.make_first(gram, name))
+gram.dfas.append(dfa)
+assert len(gram.dfas) - 1 == symbol_id - 256
 gram.start = gram.symbol_ids[self.start_symbol]
 return gram
 
@@ -162,6 +164,13 @@
 else:
 gram.labels.append(gram.symbol_ids[label])
 gram.symbol_to_label[label] = label_index
+first = self.first[label]
+if len(first) == 1:
+first, = first
+if not first[0].isupper():
+first = first.strip("\"'")
+assert label_index not in 
gram.token_to_error_string
+gram.token_to_error_string[label_index] = first
 return label_index
 elif label.isupper():
 token_index = gram.TOKENS[label]
@@ -183,7 +192,7 @@
 else:
 gram.labels.append(gram.KEYWORD_TOKEN)
 gram.keyword_ids[value] = label_index
-return label_index
+result = label_index
 else:
 try:
 token_index = gram.OPERATOR_MAP[value]
@@ -194,7 +203,10 @@
 else:
 gram.labels.append(token_index)
 gram.token_ids[token_index] = label_index
-return label_index
+result = label_index
+assert result not in gram.token_to_error_string
+gram.token_to_error_string[result] = value
+return result
 
 def make_first(self, gram, name):
 original_firsts = self.first[name]
diff --git a/pypy/interpreter/pyparser/parser.py 
b/pypy/interpreter/pyparser/parser.py
--- a/pypy/interpreter/pyparser/parser.py
+++ b/pypy/interpreter/pyparser/parser.py
@@ -1,6 +1,7 @@
 """
 A CPython inspired RPython parser.
 """
+from rpython.rlib.objectmodel import not_rpython
 
 
 class Grammar(object):
@@ -16,6 +17,7 @@
 self.symbol_names = {}
 self.symbol_to_label = {}
 self.keyword_ids = {}
+self.token_to_error_string = {}
 self.dfas = []
 self.labels = [0]
 self.token_ids = {}
@@ -41,6 +43,27 @@
 pass
 return True
 
+class DFA(object):
+def __init__(self, symbol_id, states, first):
+self.symbol_id = symbol_id
+self.states = states
+self.first = self._first_to_string(first)
+
+def could_match_token(self, label_index):
+pos = label_index >> 3
+

[pypy-commit] pypy gc-hooks: hg merge default

[antocuni]

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94191:9edf064fc152
Date: 2018-03-30 18:31 +0200
http://bitbucket.org/pypy/pypy/changeset/9edf064fc152/

Log:hg merge default

diff too long, truncating to 2000 out of 2274 lines

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -68,3 +68,14 @@
 Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify
 `W_PyCWrapperObject` which is used to call slots from the C-API, greatly
 improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks
+
+
+.. branch: fix-sre-problems
+
+Fix two (unrelated) JIT bugs manifesting in the re module:
+
+- green fields are broken and were thus disabled, plus their usage removed from
+  the _sre implementation
+
+- in rare "trace is too long" situations, the JIT could break behaviour
+  arbitrarily.
diff --git a/pypy/module/_cffi_backend/ccallback.py 
b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -232,7 +232,9 @@
 "different from the 'ffi.h' file seen at compile-time)")
 
 def py_invoke(self, ll_res, ll_args):
+key_pycode = self.key_pycode
 jitdriver1.jit_merge_point(callback=self,
+   key_pycode=key_pycode,
ll_res=ll_res,
ll_args=ll_args)
 self.do_invoke(ll_res, ll_args)
@@ -294,7 +296,7 @@
 return 'cffi_callback ' + key_pycode.get_repr()
 
 jitdriver1 = jit.JitDriver(name='cffi_callback',
-   greens=['callback.key_pycode'],
+   greens=['key_pycode'],
reds=['ll_res', 'll_args', 'callback'],
get_printable_location=get_printable_location1)
 
diff --git a/pypy/module/_io/test/test_interp_textio.py 
b/pypy/module/_io/test/test_interp_textio.py
--- a/pypy/module/_io/test/test_interp_textio.py
+++ b/pypy/module/_io/test/test_interp_textio.py
@@ -7,6 +7,11 @@
 from pypy.module._io.interp_bytesio import W_BytesIO
 from pypy.module._io.interp_textio import W_TextIOWrapper, DecodeBuffer
 
+# workaround suggestion for slowness by David McIver:
+# force hypothesis to initialize some lazy stuff
+# (which takes a lot of time, which trips the timer otherwise)
+st.text().example()
+
 def translate_newlines(text):
 text = text.replace(u'\r\n', u'\n')
 text = text.replace(u'\r', u'\n')
@@ -29,7 +34,7 @@
 
 @given(data=st_readline(),
mode=st.sampled_from(['\r', '\n', '\r\n', '']))
-@settings(deadline=None)
+@settings(deadline=None, database=None)
 def test_readline(space, data, mode):
 txt, limits = data
 w_stream = W_BytesIO(space)
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -77,15 +77,15 @@
 w_import = space.getattr(w_builtin, space.newtext("__import__"))
 return space.call_function(w_import, space.newtext("re"))
 
-def matchcontext(space, ctx):
+def matchcontext(space, ctx, pattern):
 try:
-return rsre_core.match_context(ctx)
+return rsre_core.match_context(ctx, pattern)
 except rsre_core.Error as e:
 raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
-def searchcontext(space, ctx):
+def searchcontext(space, ctx, pattern):
 try:
-return rsre_core.search_context(ctx)
+return rsre_core.search_context(ctx, pattern)
 except rsre_core.Error as e:
 raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
@@ -114,7 +114,7 @@
 pos = len(unicodestr)
 if endpos > len(unicodestr):
 endpos = len(unicodestr)
-return rsre_core.UnicodeMatchContext(self.code, unicodestr,
+return rsre_core.UnicodeMatchContext(unicodestr,
  pos, endpos, self.flags)
 elif space.isinstance_w(w_string, space.w_bytes):
 str = space.bytes_w(w_string)
@@ -122,7 +122,7 @@
 pos = len(str)
 if endpos > len(str):
 endpos = len(str)
-return rsre_core.StrMatchContext(self.code, str,
+return rsre_core.StrMatchContext(str,
  pos, endpos, self.flags)
 else:
 buf = space.readbuf_w(w_string)
@@ -132,7 +132,7 @@
 pos = size
 if endpos > size:
 endpos = size
-return rsre_core.BufMatchContext(self.code, buf,
+return rsre_core.BufMatchContext(buf,
  pos, endpos, self.flags)
 
 def getmatch(self, ctx, found):
@@ -144,12 +144,12 @@
 @unwrap_spec(pos=int, endpos=int)
 def match_w(self, w_string, pos=0,