[pypy-commit] pypy mmap-for-arenas: A branch to try to use mmap() instead of malloc() for arenas from the GC

2017-11-30 Thread arigo 
Author: Armin Rigo 
Branch: mmap-for-arenas
Changeset: r93217:75f5a5c594a8
Date: 2017-11-30 16:54 +0100
http://bitbucket.org/pypy/pypy/changeset/75f5a5c594a8/

Log:A branch to try to use mmap() instead of malloc() for arenas from
the GC

___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy mmap-for-arenas: (fijal, arigo)

2017-11-30 Thread arigo 
Author: Armin Rigo 
Branch: mmap-for-arenas
Changeset: r93218:6412ce4e1198
Date: 2017-11-30 17:30 +0100
http://bitbucket.org/pypy/pypy/changeset/6412ce4e1198/

Log:(fijal, arigo)

Trying to use mmap() to allocate arenas

diff --git a/rpython/memory/gc/minimarkpage.py 
b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -292,7 +292,7 @@
 #
 # 'arena_base' points to the start of malloced memory; it might not
 # be a page-aligned address
-arena_base = llarena.arena_malloc(self.arena_size, False)
+arena_base = llarena.arena_mmap(self.arena_size)
 if not arena_base:
 out_of_memory("out of memory: couldn't allocate the next arena")
 arena_end = arena_base + self.arena_size
@@ -395,8 +395,7 @@
 if arena.nfreepages == arena.totalpages:
 #
 # The whole arena is empty.  Free it.
-llarena.arena_reset(arena.base, self.arena_size, 4)
-llarena.arena_free(arena.base)
+llarena.arena_munmap(arena.base, self.arena_size)
 lltype.free(arena, flavor='raw', track_allocation=False)
 #
 else:
diff --git a/rpython/rtyper/lltypesystem/llarena.py 
b/rpython/rtyper/lltypesystem/llarena.py
--- a/rpython/rtyper/lltypesystem/llarena.py
+++ b/rpython/rtyper/lltypesystem/llarena.py
@@ -327,6 +327,16 @@
 assert not arena_addr.arena.objectptrs
 arena_addr.arena.mark_freed()
 
+def arena_mmap(nbytes):
+"""Allocate and return a new arena, zero-initialized by the
+system, calling mmap()."""
+return arena_malloc(nbytes, True)
+
+def arena_munmap(arena_addr):
+"""Release an arena allocated with arena_mmap()."""
+arena_free(arena_addr)
+
+
 def arena_reset(arena_addr, size, zero):
 """Free all objects in the arena, which can then be reused.
 This can also be used on a subrange of the arena.
@@ -530,6 +540,30 @@
   llfakeimpl=arena_free,
   sandboxsafe=True)
 
+def llimpl_arena_mmap(nbytes):
+from rpython.rlib import rmmap
+flags = rmmap.MAP_PRIVATE | rmmap.MAP_ANONYMOUS
+prot = rmmap.PROT_READ | rmmap.PROT_WRITE
+p = rffi.cast(llmemory.Address, rmmap.c_mmap_safe(
+lltype.nullptr(rmmap.PTR.TO), nbytes, prot, flags, -1, 0))
+if p == rffi.cast(llmemory.Address, -1):
+p = rffi.cast(llmemory.Address, 0)
+return p
+register_external(arena_mmap, [int], llmemory.Address,
+  'll_arena.arena_mmap',
+  llimpl=llimpl_arena_mmap,
+  llfakeimpl=arena_mmap,
+  sandboxsafe=True)
+
+def llimpl_arena_munmap(arena_addr, nbytes):
+from rpython.rlib import rmmap
+rmmap.c_munmap_safe(rffi.cast(rmmap.PTR, arena_addr), nbytes)
+register_external(arena_munmap, [llmemory.Address, int], None,
+  'll_arena.arena_munmap',
+  llimpl=llimpl_arena_munmap,
+  llfakeimpl=arena_munmap,
+  sandboxsafe=True)
+
 def llimpl_arena_reset(arena_addr, size, zero):
 if zero:
 if zero == 1:
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy mmap-for-arenas: test fix

2017-11-30 Thread arigo 
Author: Armin Rigo 
Branch: mmap-for-arenas
Changeset: r93219:4c75975c98db
Date: 2017-11-30 17:37 +0100
http://bitbucket.org/pypy/pypy/changeset/4c75975c98db/

Log:test fix

diff --git a/rpython/rtyper/lltypesystem/llarena.py 
b/rpython/rtyper/lltypesystem/llarena.py
--- a/rpython/rtyper/lltypesystem/llarena.py
+++ b/rpython/rtyper/lltypesystem/llarena.py
@@ -332,9 +332,10 @@
 system, calling mmap()."""
 return arena_malloc(nbytes, True)
 
-def arena_munmap(arena_addr):
+def arena_munmap(arena_addr, nbytes):
 """Release an arena allocated with arena_mmap()."""
 arena_free(arena_addr)
+assert nbytes == arena_addr.arena.nbytes
 
 
 def arena_reset(arena_addr, size, zero):
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: backout changes that broke translation in unclear ways (thanks RPython)

2017-11-30 Thread fijal 
Author: fijal
Branch: 
Changeset: r93220:30c6fda0a499
Date: 2017-11-30 18:38 +0200
http://bitbucket.org/pypy/pypy/changeset/30c6fda0a499/

Log:backout changes that broke translation in unclear ways (thanks
RPython)

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py 
b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -25,6 +25,19 @@
 return (1 << ((byte_size << 3) - 1)) - 1
 
 
+IS_64_BIT = sys.maxint > 2**32
+
+def next_pow2_m1(n):
+"""Calculate next power of 2 greater than n minus one."""
+n |= n >> 1
+n |= n >> 2
+n |= n >> 4
+n |= n >> 8
+n |= n >> 16
+if IS_64_BIT:
+n |= n >> 32
+return n
+
 
 class OptIntBounds(Optimization):
 """Keeps track of the bounds placed on integers by guards and remove
@@ -37,7 +50,7 @@
 return dispatch_postprocess(self, op)
 
 def propagate_bounds_backward(self, box):
-# FIXME: This takes care of the instruction where box is the result
+# FIXME: This takes care of the instruction where box is the reuslt
 #but the bounds produced by all instructions where box is
 #an argument might also be tighten
 b = self.getintbound(box)
@@ -78,8 +91,14 @@
 b1 = self.getintbound(v1)
 v2 = self.get_box_replacement(op.getarg(1))
 b2 = self.getintbound(v2)
-b = b1.or_bound(b2)
-self.getintbound(op).intersect(b)
+if b1.known_ge(IntBound(0, 0)) and \
+   b2.known_ge(IntBound(0, 0)):
+r = self.getintbound(op)
+if b1.has_upper and b2.has_upper:
+mostsignificant = b1.upper | b2.upper
+r.intersect(IntBound(0, next_pow2_m1(mostsignificant)))
+else:
+r.make_ge(IntBound(0, 0))
 
 optimize_INT_OR = optimize_INT_OR_or_XOR
 optimize_INT_XOR = optimize_INT_OR_or_XOR
@@ -93,8 +112,15 @@
 def postprocess_INT_AND(self, op):
 b1 = self.getintbound(op.getarg(0))
 b2 = self.getintbound(op.getarg(1))
-b = b1.and_bound(b2)
-self.getintbound(op).intersect(b)
+r = self.getintbound(op)
+pos1 = b1.known_ge(IntBound(0, 0))
+pos2 = b2.known_ge(IntBound(0, 0))
+if pos1 or pos2:
+r.make_ge(IntBound(0, 0))
+if pos1:
+r.make_le(b1)
+if pos2:
+r.make_le(b2)
 
 def optimize_INT_SUB(self, op):
 return self.emit(op)
@@ -185,10 +211,16 @@
 r.intersect(b1.py_div_bound(b2))
 
 def post_call_INT_PY_MOD(self, op):
-b1 = self.getintbound(op.getarg(1))
 b2 = self.getintbound(op.getarg(2))
-r = self.getintbound(op)
-r.intersect(b1.mod_bound(b2))
+if b2.is_constant():
+val = b2.getint()
+r = self.getintbound(op)
+if val >= 0:# with Python's modulo:  0 <= (x % pos) < pos
+r.make_ge(IntBound(0, 0))
+r.make_lt(IntBound(val, val))
+else:   # with Python's modulo:  neg < (x % neg) <= 0
+r.make_gt(IntBound(val, val))
+r.make_le(IntBound(0, 0))
 
 def optimize_INT_LSHIFT(self, op):
 return self.emit(op)
@@ -404,7 +436,7 @@
 
 def optimize_INT_FORCE_GE_ZERO(self, op):
 b = self.getintbound(op.getarg(0))
-if b.known_nonnegative():
+if b.known_ge(IntBound(0, 0)):
 self.make_equal_to(op, op.getarg(0))
 else:
 return self.emit(op)
@@ -615,7 +647,7 @@
 if r.is_constant():
 if r.getint() == valnonzero:
 b1 = self.getintbound(op.getarg(0))
-if b1.known_nonnegative():
+if b1.known_ge(IntBound(0, 0)):
 b1.make_gt(IntBound(0, 0))
 self.propagate_bounds_backward(op.getarg(0))
 elif r.getint() == valzero:
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py 
b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -12,19 +12,6 @@
 MAXINT = maxint
 MININT = -maxint - 1
 
-IS_64_BIT = sys.maxint > 2**32
-
-def next_pow2_m1(n):
-"""Calculate next power of 2 greater than n minus one."""
-n |= n >> 1
-n |= n >> 2
-n |= n >> 4
-n |= n >> 8
-n |= n >> 16
-if IS_64_BIT:
-n |= n >> 32
-return n
-
 
 class IntBound(AbstractInfo):
 _attrs_ = ('has_upper', 'has_lower', 'upper', 'lower')
@@ -105,9 +92,6 @@
 def known_ge(self, other):
 return other.known_le(self)
 
-def known_nonnegative(self):
-return self.has_lower and 0 <= self.lower
-
 def intersect(self, other):
 r = False
 
@@ -208,22 +192,10 @@
 else:
 return IntUnbounded()
 
-def mod_bound(self, other):
-

[pypy-commit] pypy mmap-for-arenas: merge default

2017-11-30 Thread fijal 
Author: fijal
Branch: mmap-for-arenas
Changeset: r93221:6fb9f1a724da
Date: 2017-11-30 18:38 +0200
http://bitbucket.org/pypy/pypy/changeset/6fb9f1a724da/

Log:merge default

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py 
b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -25,6 +25,19 @@
 return (1 << ((byte_size << 3) - 1)) - 1
 
 
+IS_64_BIT = sys.maxint > 2**32
+
+def next_pow2_m1(n):
+"""Calculate next power of 2 greater than n minus one."""
+n |= n >> 1
+n |= n >> 2
+n |= n >> 4
+n |= n >> 8
+n |= n >> 16
+if IS_64_BIT:
+n |= n >> 32
+return n
+
 
 class OptIntBounds(Optimization):
 """Keeps track of the bounds placed on integers by guards and remove
@@ -37,7 +50,7 @@
 return dispatch_postprocess(self, op)
 
 def propagate_bounds_backward(self, box):
-# FIXME: This takes care of the instruction where box is the result
+# FIXME: This takes care of the instruction where box is the reuslt
 #but the bounds produced by all instructions where box is
 #an argument might also be tighten
 b = self.getintbound(box)
@@ -78,8 +91,14 @@
 b1 = self.getintbound(v1)
 v2 = self.get_box_replacement(op.getarg(1))
 b2 = self.getintbound(v2)
-b = b1.or_bound(b2)
-self.getintbound(op).intersect(b)
+if b1.known_ge(IntBound(0, 0)) and \
+   b2.known_ge(IntBound(0, 0)):
+r = self.getintbound(op)
+if b1.has_upper and b2.has_upper:
+mostsignificant = b1.upper | b2.upper
+r.intersect(IntBound(0, next_pow2_m1(mostsignificant)))
+else:
+r.make_ge(IntBound(0, 0))
 
 optimize_INT_OR = optimize_INT_OR_or_XOR
 optimize_INT_XOR = optimize_INT_OR_or_XOR
@@ -93,8 +112,15 @@
 def postprocess_INT_AND(self, op):
 b1 = self.getintbound(op.getarg(0))
 b2 = self.getintbound(op.getarg(1))
-b = b1.and_bound(b2)
-self.getintbound(op).intersect(b)
+r = self.getintbound(op)
+pos1 = b1.known_ge(IntBound(0, 0))
+pos2 = b2.known_ge(IntBound(0, 0))
+if pos1 or pos2:
+r.make_ge(IntBound(0, 0))
+if pos1:
+r.make_le(b1)
+if pos2:
+r.make_le(b2)
 
 def optimize_INT_SUB(self, op):
 return self.emit(op)
@@ -185,10 +211,16 @@
 r.intersect(b1.py_div_bound(b2))
 
 def post_call_INT_PY_MOD(self, op):
-b1 = self.getintbound(op.getarg(1))
 b2 = self.getintbound(op.getarg(2))
-r = self.getintbound(op)
-r.intersect(b1.mod_bound(b2))
+if b2.is_constant():
+val = b2.getint()
+r = self.getintbound(op)
+if val >= 0:# with Python's modulo:  0 <= (x % pos) < pos
+r.make_ge(IntBound(0, 0))
+r.make_lt(IntBound(val, val))
+else:   # with Python's modulo:  neg < (x % neg) <= 0
+r.make_gt(IntBound(val, val))
+r.make_le(IntBound(0, 0))
 
 def optimize_INT_LSHIFT(self, op):
 return self.emit(op)
@@ -404,7 +436,7 @@
 
 def optimize_INT_FORCE_GE_ZERO(self, op):
 b = self.getintbound(op.getarg(0))
-if b.known_nonnegative():
+if b.known_ge(IntBound(0, 0)):
 self.make_equal_to(op, op.getarg(0))
 else:
 return self.emit(op)
@@ -615,7 +647,7 @@
 if r.is_constant():
 if r.getint() == valnonzero:
 b1 = self.getintbound(op.getarg(0))
-if b1.known_nonnegative():
+if b1.known_ge(IntBound(0, 0)):
 b1.make_gt(IntBound(0, 0))
 self.propagate_bounds_backward(op.getarg(0))
 elif r.getint() == valzero:
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py 
b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -12,19 +12,6 @@
 MAXINT = maxint
 MININT = -maxint - 1
 
-IS_64_BIT = sys.maxint > 2**32
-
-def next_pow2_m1(n):
-"""Calculate next power of 2 greater than n minus one."""
-n |= n >> 1
-n |= n >> 2
-n |= n >> 4
-n |= n >> 8
-n |= n >> 16
-if IS_64_BIT:
-n |= n >> 32
-return n
-
 
 class IntBound(AbstractInfo):
 _attrs_ = ('has_upper', 'has_lower', 'upper', 'lower')
@@ -105,9 +92,6 @@
 def known_ge(self, other):
 return other.known_le(self)
 
-def known_nonnegative(self):
-return self.has_lower and 0 <= self.lower
-
 def intersect(self, other):
 r = False
 
@@ -208,22 +192,10 @@
 else:
 return IntUnbounded()
 
-def mod_bound(self, other):
-r = IntUnbounded()
-if other.is_constant()

[pypy-commit] pypy mmap-for-arenas: merge

2017-11-30 Thread fijal 
Author: fijal
Branch: mmap-for-arenas
Changeset: r93222:2e594f3e5237
Date: 2017-11-30 18:44 +0200
http://bitbucket.org/pypy/pypy/changeset/2e594f3e5237/

Log:merge

diff --git a/rpython/rtyper/lltypesystem/llarena.py 
b/rpython/rtyper/lltypesystem/llarena.py
--- a/rpython/rtyper/lltypesystem/llarena.py
+++ b/rpython/rtyper/lltypesystem/llarena.py
@@ -332,9 +332,10 @@
 system, calling mmap()."""
 return arena_malloc(nbytes, True)
 
-def arena_munmap(arena_addr):
+def arena_munmap(arena_addr, nbytes):
 """Release an arena allocated with arena_mmap()."""
 arena_free(arena_addr)
+assert nbytes == arena_addr.arena.nbytes
 
 
 def arena_reset(arena_addr, size, zero):
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: Re-revert 30c6fda0a499, and add the proper fix, hopefully

2017-11-30 Thread arigo 
Author: Armin Rigo 
Branch: 
Changeset: r93223:e6c7a428f649
Date: 2017-11-30 17:48 +0100
http://bitbucket.org/pypy/pypy/changeset/e6c7a428f649/

Log:Re-revert 30c6fda0a499, and add the proper fix, hopefully

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py 
b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -25,19 +25,6 @@
 return (1 << ((byte_size << 3) - 1)) - 1
 
 
-IS_64_BIT = sys.maxint > 2**32
-
-def next_pow2_m1(n):
-"""Calculate next power of 2 greater than n minus one."""
-n |= n >> 1
-n |= n >> 2
-n |= n >> 4
-n |= n >> 8
-n |= n >> 16
-if IS_64_BIT:
-n |= n >> 32
-return n
-
 
 class OptIntBounds(Optimization):
 """Keeps track of the bounds placed on integers by guards and remove
@@ -50,7 +37,7 @@
 return dispatch_postprocess(self, op)
 
 def propagate_bounds_backward(self, box):
-# FIXME: This takes care of the instruction where box is the reuslt
+# FIXME: This takes care of the instruction where box is the result
 #but the bounds produced by all instructions where box is
 #an argument might also be tighten
 b = self.getintbound(box)
@@ -91,14 +78,8 @@
 b1 = self.getintbound(v1)
 v2 = self.get_box_replacement(op.getarg(1))
 b2 = self.getintbound(v2)
-if b1.known_ge(IntBound(0, 0)) and \
-   b2.known_ge(IntBound(0, 0)):
-r = self.getintbound(op)
-if b1.has_upper and b2.has_upper:
-mostsignificant = b1.upper | b2.upper
-r.intersect(IntBound(0, next_pow2_m1(mostsignificant)))
-else:
-r.make_ge(IntBound(0, 0))
+b = b1.or_bound(b2)
+self.getintbound(op).intersect(b)
 
 optimize_INT_OR = optimize_INT_OR_or_XOR
 optimize_INT_XOR = optimize_INT_OR_or_XOR
@@ -112,15 +93,8 @@
 def postprocess_INT_AND(self, op):
 b1 = self.getintbound(op.getarg(0))
 b2 = self.getintbound(op.getarg(1))
-r = self.getintbound(op)
-pos1 = b1.known_ge(IntBound(0, 0))
-pos2 = b2.known_ge(IntBound(0, 0))
-if pos1 or pos2:
-r.make_ge(IntBound(0, 0))
-if pos1:
-r.make_le(b1)
-if pos2:
-r.make_le(b2)
+b = b1.and_bound(b2)
+self.getintbound(op).intersect(b)
 
 def optimize_INT_SUB(self, op):
 return self.emit(op)
@@ -211,16 +185,10 @@
 r.intersect(b1.py_div_bound(b2))
 
 def post_call_INT_PY_MOD(self, op):
+b1 = self.getintbound(op.getarg(1))
 b2 = self.getintbound(op.getarg(2))
-if b2.is_constant():
-val = b2.getint()
-r = self.getintbound(op)
-if val >= 0:# with Python's modulo:  0 <= (x % pos) < pos
-r.make_ge(IntBound(0, 0))
-r.make_lt(IntBound(val, val))
-else:   # with Python's modulo:  neg < (x % neg) <= 0
-r.make_gt(IntBound(val, val))
-r.make_le(IntBound(0, 0))
+r = self.getintbound(op)
+r.intersect(b1.mod_bound(b2))
 
 def optimize_INT_LSHIFT(self, op):
 return self.emit(op)
@@ -436,7 +404,7 @@
 
 def optimize_INT_FORCE_GE_ZERO(self, op):
 b = self.getintbound(op.getarg(0))
-if b.known_ge(IntBound(0, 0)):
+if b.known_nonnegative():
 self.make_equal_to(op, op.getarg(0))
 else:
 return self.emit(op)
@@ -647,7 +615,7 @@
 if r.is_constant():
 if r.getint() == valnonzero:
 b1 = self.getintbound(op.getarg(0))
-if b1.known_ge(IntBound(0, 0)):
+if b1.known_nonnegative():
 b1.make_gt(IntBound(0, 0))
 self.propagate_bounds_backward(op.getarg(0))
 elif r.getint() == valzero:
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py 
b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -12,6 +12,19 @@
 MAXINT = maxint
 MININT = -maxint - 1
 
+IS_64_BIT = sys.maxint > 2**32
+
+def next_pow2_m1(n):
+"""Calculate next power of 2 greater than n minus one."""
+n |= n >> 1
+n |= n >> 2
+n |= n >> 4
+n |= n >> 8
+n |= n >> 16
+if IS_64_BIT:
+n |= n >> 32
+return n
+
 
 class IntBound(AbstractInfo):
 _attrs_ = ('has_upper', 'has_lower', 'upper', 'lower')
@@ -92,6 +105,9 @@
 def known_ge(self, other):
 return other.known_le(self)
 
+def known_nonnegative(self):
+return self.has_lower and 0 <= self.lower
+
 def intersect(self, other):
 r = False
 
@@ -192,10 +208,22 @@
 else:
 return IntUnbounded()
 
+def mod_bound(self, other):
+r = IntUnbounde

[pypy-commit] pypy mmap-for-arenas: translation fix

2017-11-30 Thread arigo 
Author: Armin Rigo 
Branch: mmap-for-arenas
Changeset: r93224:40ad6dbda37b
Date: 2017-11-30 18:18 +0100
http://bitbucket.org/pypy/pypy/changeset/40ad6dbda37b/

Log:translation fix

diff --git a/rpython/rtyper/lltypesystem/llarena.py 
b/rpython/rtyper/lltypesystem/llarena.py
--- a/rpython/rtyper/lltypesystem/llarena.py
+++ b/rpython/rtyper/lltypesystem/llarena.py
@@ -558,6 +558,7 @@
 
 def llimpl_arena_munmap(arena_addr, nbytes):
 from rpython.rlib import rmmap
+assert nbytes >= 0
 rmmap.c_munmap_safe(rffi.cast(rmmap.PTR, arena_addr), nbytes)
 register_external(arena_munmap, [llmemory.Address, int], None,
   'll_arena.arena_munmap',
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy memory-accounting: merge mmap-for-llarena

2017-11-30 Thread fijal 
Author: fijal
Branch: memory-accounting
Changeset: r93225:5ba0bf0bf684
Date: 2017-11-30 20:49 +0200
http://bitbucket.org/pypy/pypy/changeset/5ba0bf0bf684/

Log:merge mmap-for-llarena

diff too long, truncating to 2000 out of 18247 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -71,6 +71,8 @@
 ^lib_pypy/.+.c$
 ^lib_pypy/.+.o$
 ^lib_pypy/.+.so$
+^lib_pypy/.+.pyd$
+^lib_pypy/Release/
 ^pypy/doc/discussion/.+\.html$
 ^include/.+\.h$
 ^include/.+\.inl$
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -40,3 +40,7 @@
 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1
 c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0
 a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0
+03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0
+d72f9800a42b46a8056951b1da2426d2c2d8d502 release-pypy3.5-v5.9.0
+03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0
+84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
--- a/_pytest/terminal.py
+++ b/_pytest/terminal.py
@@ -366,11 +366,11 @@
 EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, EXIT_USAGEERROR,
 EXIT_NOTESTSCOLLECTED)
 if exitstatus in summary_exit_codes:
-self.config.hook.pytest_terminal_summary(terminalreporter=self)
 self.summary_errors()
 self.summary_failures()
 self.summary_warnings()
 self.summary_passes()
+self.config.hook.pytest_terminal_summary(terminalreporter=self)
 if exitstatus == EXIT_INTERRUPTED:
 self._report_keyboardinterrupt()
 del self._keyboardinterrupt_memo
diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt
new file mode 100644
--- /dev/null
+++ b/extra_tests/requirements.txt
@@ -0,0 +1,2 @@
+pytest
+hypothesis
diff --git a/extra_tests/test_bytes.py b/extra_tests/test_bytes.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/test_bytes.py
@@ -0,0 +1,84 @@
+from hypothesis import strategies as st
+from hypothesis import given, example
+
+st_bytestring = st.binary() | st.binary().map(bytearray)
+
+@given(st_bytestring, st_bytestring, st_bytestring)
+def test_find(u, prefix, suffix):
+s = prefix + u + suffix
+assert 0 <= s.find(u) <= len(prefix)
+assert s.find(u, len(prefix), len(s) - len(suffix)) == len(prefix)
+
+@given(st_bytestring, st_bytestring, st_bytestring)
+def test_index(u, prefix, suffix):
+s = prefix + u + suffix
+assert 0 <= s.index(u) <= len(prefix)
+assert s.index(u, len(prefix), len(s) - len(suffix)) == len(prefix)
+
+@given(st_bytestring, st_bytestring, st_bytestring)
+def test_rfind(u, prefix, suffix):
+s = prefix + u + suffix
+assert s.rfind(u) >= len(prefix)
+assert s.rfind(u, len(prefix), len(s) - len(suffix)) == len(prefix)
+
+@given(st_bytestring, st_bytestring, st_bytestring)
+def test_rindex(u, prefix, suffix):
+s = prefix + u + suffix
+assert s.rindex(u) >= len(prefix)
+assert s.rindex(u, len(prefix), len(s) - len(suffix)) == len(prefix)
+
+def adjust_indices(u, start, end):
+if end < 0:
+end = max(end + len(u), 0)
+else:
+end = min(end, len(u))
+if start < 0:
+start = max(start + len(u), 0)
+return start, end
+
+@given(st_bytestring, st_bytestring)
+def test_startswith_basic(u, v):
+assert u.startswith(v) is (u[:len(v)] == v)
+
+@example(b'x', b'', 1)
+@example(b'x', b'', 2)
+@given(st_bytestring, st_bytestring, st.integers())
+def test_startswith_start(u, v, start):
+expected = u[start:].startswith(v) if v else (start <= len(u))
+assert u.startswith(v, start) is expected
+
+@example(b'x', b'', 1, 0)
+@example(b'xx', b'', -1, 0)
+@given(st_bytestring, st_bytestring, st.integers(), st.integers())
+def test_startswith_3(u, v, start, end):
+if v:
+expected = u[start:end].startswith(v)
+else:  # CPython leaks implementation details in this case
+start0, end0 = adjust_indices(u, start, end)
+expected = start0 <= len(u) and start0 <= end0
+assert u.startswith(v, start, end) is expected
+
+@given(st_bytestring, st_bytestring)
+def test_endswith_basic(u, v):
+if len(v) > len(u):
+assert u.endswith(v) is False
+else:
+assert u.endswith(v) is (u[len(u) - len(v):] == v)
+
+@example(b'x', b'', 1)
+@example(b'x', b'', 2)
+@given(st_bytestring, st_bytestring, st.integers())
+def test_endswith_2(u, v, start):
+expected = u[start:].endswith(v) if v else (start <= len(u))
+assert u.endswith(v, start) is expected
+
+@example(b'x', b'', 1, 0)
+@example(b'xx', b'', -1, 0)
+@given(st_bytestring, st_bytestring, st.integers(), st.integers())
+def test_endswith_3(u, v, start, end):
+if v:
+expected = u[start:end].endswith(v)
+else:  # CPython leaks implementation details in this case
+start0, end0 = adjust_indices(u, start, end)
+ex

[pypy-commit] pypy py3.5: Module names inside a zip are not fsencoded - they can be any str

2017-11-30 Thread rlamy 
Author: Ronan Lamy 
Branch: py3.5
Changeset: r93226:f1a55693
Date: 2017-11-30 19:39 +
http://bitbucket.org/pypy/pypy/changeset/f1a55693/

Log:Module names inside a zip are not fsencoded - they can be any str

diff --git a/pypy/module/zipimport/interp_zipimport.py 
b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -47,7 +47,7 @@
 # THIS IS A TERRIBLE HACK TO BE CPYTHON COMPATIBLE
 
 def getitem(self, space, w_name):
-return self._getitem(space, space.fsencode_w(w_name))
+return self._getitem(space, space.text_w(w_name))
 
 def _getitem(self, space, name):
 try:
@@ -90,14 +90,14 @@
 def iteritems(self, space):
 return space.iter(self.items(space))
 
-@unwrap_spec(name='fsencode')
+@unwrap_spec(name='text')
 def contains(self, space, name):
 return space.newbool(name in self.cache)
 
 def clear(self, space):
 self.cache = {}
 
-@unwrap_spec(name='fsencode')
+@unwrap_spec(name='text')
 def delitem(self, space, name):
 del self.cache[name]
 
@@ -221,7 +221,7 @@
 except KeyError:
 return False
 
-@unwrap_spec(fullname='fsencode')
+@unwrap_spec(fullname='text')
 def find_module(self, space, fullname, w_path=None):
 filename = self.make_filename(fullname)
 for _, _, ext in ENUMERATE_EXTS:
@@ -247,7 +247,7 @@
 return self.filename + os.path.sep + filename
 
 def load_module(self, space, w_fullname):
-fullname = space.fsencode_w(w_fullname)
+fullname = space.text_w(w_fullname)
 filename = self.make_filename(fullname)
 for compiled, is_package, ext in ENUMERATE_EXTS:
 fname = filename + ext
@@ -287,7 +287,7 @@
 raise
 raise oefmt(get_error(space), "can't find module %R", w_fullname)
 
-@unwrap_spec(filename='fsencode')
+@unwrap_spec(filename='text')
 def get_data(self, space, filename):
 filename = self._find_relative_path(filename)
 try:
@@ -301,7 +301,7 @@
 raise zlib_error(space, e.msg)
 
 def get_code(self, space, w_fullname):
-fullname = space.fsencode_w(w_fullname)
+fullname = space.text_w(w_fullname)
 filename = self.make_filename(fullname)
 for compiled, _, ext in ENUMERATE_EXTS:
 if self.have_modulefile(space, filename + ext):
@@ -325,7 +325,7 @@
 "Cannot find source or code for %R in %R",
 w_fullname, space.newfilename(self.name))
 
-@unwrap_spec(fullname='fsencode')
+@unwrap_spec(fullname='text')
 def get_source(self, space, fullname):
 filename = self.make_filename(fullname)
 found = False
@@ -348,7 +348,7 @@
 space.newfilename(self.name))
 
 def get_filename(self, space, w_fullname):
-fullname = space.fsencode_w(w_fullname)
+fullname = space.text_w(w_fullname)
 filename = self.make_filename(fullname)
 for _, is_package, ext in ENUMERATE_EXTS:
 if self.have_modulefile(space, filename + ext):
@@ -360,7 +360,7 @@
 space.newfilename(self.name))
 
 def is_package(self, space, w_fullname):
-fullname = space.fsencode_w(w_fullname)
+fullname = space.text_w(w_fullname)
 filename = self.make_filename(fullname)
 for _, is_package, ext in ENUMERATE_EXTS:
 if self.have_modulefile(space, filename + ext):
@@ -385,7 +385,7 @@
 return True, self.filename + os.path.sep + self.corr_zname(dirpath)
 return False, None
 
-@unwrap_spec(fullname='fsencode')
+@unwrap_spec(fullname='text')
 def find_loader(self, space, fullname, w_path=None):
 found, ns_portion = self._find_loader(space, fullname)
 if not found:
@@ -401,9 +401,9 @@
 name = space.fsencode_w(w_name)
 ok = False
 parts_ends = [i for i in range(0, len(name))
-if name[i] == os.path.sep or name[i] == ZIPSEP]
+if name[i] == os.path.sep or name[i] == ZIPSEP]
 parts_ends.append(len(name))
-filename = "" # make annotator happy
+filename = ""  # make annotator happy
 for i in parts_ends:
 filename = name[:i]
 if not filename:
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy unicode-utf8: I think this is a speed-up

2017-11-30 Thread arigo 
Author: Armin Rigo 
Branch: unicode-utf8
Changeset: r93227:91d2d71881e2
Date: 2017-11-30 21:43 +0100
http://bitbucket.org/pypy/pypy/changeset/91d2d71881e2/

Log:I think this is a speed-up

diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -86,8 +86,8 @@
 """Gives the position of the next codepoint after pos.
 Assumes valid utf8.  'pos' must be before the end of the string.
 """
+assert pos >= 0
 chr1 = ord(code[pos])
-assert pos >= 0
 if chr1 <= 0x7F:
 return pos + 1
 if chr1 <= 0xDF:
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.5: .pyo suffix is meaningless now (PEP 488)

2017-11-30 Thread rlamy 
Author: Ronan Lamy 
Branch: py3.5
Changeset: r93228:f8e7ad765a37
Date: 2017-12-01 02:06 +
http://bitbucket.org/pypy/pypy/changeset/f8e7ad765a37/

Log:.pyo suffix is meaningless now (PEP 488)

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -729,7 +729,7 @@
 SourceFileLoader, SourcelessFileLoader)
 if IS_WINDOWS:
 filename = filename.lower()
-if filename.endswith('.pyc') or filename.endswith('.pyo'):
+if filename.endswith('.pyc'):
 # We don't actually load via SourcelessFileLoader
 # because '__main__' must not be listed inside
 # 'importlib._bootstrap._module_locks' (it deadlocks
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -254,7 +254,7 @@
 assert typ == imp.PY_SOURCE
 source = file.read()
 file.close()
-if fn.endswith('.pyc') or fn.endswith('.pyo'):
+if fn.endswith('.pyc'):
 fn = fn[:-1]
 app = gateway.applevel(source, filename=fn, modname=appname)
 applevelcache[impbase] = app
diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py
--- a/pypy/module/imp/interp_imp.py
+++ b/pypy/module/imp/interp_imp.py
@@ -27,7 +27,7 @@
 
 def get_tag(space):
 """get_tag() -> string
-Return the magic tag for .pyc or .pyo files."""
+Return the magic tag for .pyc files."""
 return space.newtext(importing.PYC_TAG)
 
 def get_file(space, w_file, filename, filemode):
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -85,7 +85,7 @@
 assert suffix == '.py'
 assert mode == 'r'
 elif type == imp.PY_COMPILED:
-assert suffix in ('.pyc', '.pyo')
+assert suffix == '.pyc'
 assert mode == 'rb'
 elif type == imp.C_EXTENSION:
 assert suffix.endswith(('.pyd', '.so'))
diff --git a/pypy/module/zipimport/interp_zipimport.py 
b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -18,10 +18,8 @@
 
 ENUMERATE_EXTS = unrolling_iterable(
 [(True, True, ZIPSEP + '__init__.pyc'),
- (True, True, ZIPSEP + '__init__.pyo'),
  (False, True, ZIPSEP + '__init__.py'),
  (True, False, '.pyc'),
- (True, False, '.pyo'),
  (False, False, '.py')])
 
 class Cache:
diff --git a/pypy/sandbox/pypy_interact.py b/pypy/sandbox/pypy_interact.py
--- a/pypy/sandbox/pypy_interact.py
+++ b/pypy/sandbox/pypy_interact.py
@@ -46,7 +46,7 @@
 # * can access its own executable
 # * can access the pure Python libraries
 # * can access the temporary usession directory as /tmp
-exclude = ['.pyc', '.pyo']
+exclude = ['.pyc']
 if self.tmpdir is None:
 tmpdirnode = Dir({})
 else:
@@ -57,7 +57,7 @@
 'bin': Dir({
 'pypy3-c': RealFile(self.executable, mode=0111),
 'lib-python': RealDir(os.path.join(libroot, 'lib-python'),
-  exclude=exclude), 
+  exclude=exclude),
 'lib_pypy': RealDir(os.path.join(libroot, 'lib_pypy'),
   exclude=exclude),
 }),
@@ -66,7 +66,7 @@
 
 def main():
 from getopt import getopt  # and not gnu_getopt!
-options, arguments = getopt(sys.argv[1:], 't:hv', 
+options, arguments = getopt(sys.argv[1:], 't:hv',
 ['tmp=', 'heapsize=', 'timeout=', 'log=',
  'verbose', 'help'])
 tmpdir = None
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.5: Module names inside a zip are not fsencoded, part 2

2017-11-30 Thread rlamy 
Author: Ronan Lamy 
Branch: py3.5
Changeset: r93229:8309e6092c02
Date: 2017-12-01 02:25 +
http://bitbucket.org/pypy/pypy/changeset/8309e6092c02/

Log:Module names inside a zip are not fsencoded, part 2

diff --git a/pypy/module/zipimport/interp_zipimport.py 
b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -1,3 +1,5 @@
+import os
+import stat
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
@@ -9,8 +11,6 @@
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.rzipfile import RZipFile, BadZipfile
 from rpython.rlib.rzlib import RZlibError
-import os
-import stat
 
 ZIPSEP = '/'
 # note that zipfiles always use slash, but for OSes with other
@@ -145,7 +145,7 @@
 return fname
 
 def import_py_file(self, space, modname, filename, buf, pkgpath):
-w_mod = Module(space, space.newfilename(modname))
+w_mod = Module(space, space.newtext(modname))
 real_name = self.filename + os.path.sep + self.corr_zname(filename)
 space.setattr(w_mod, space.newtext('__loader__'), self)
 importing._prepare_module(space, w_mod, real_name, pkgpath)
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.5: hg merge default

2017-11-30 Thread rlamy 
Author: Ronan Lamy 
Branch: py3.5
Changeset: r93230:c932756506d4
Date: 2017-12-01 02:26 +
http://bitbucket.org/pypy/pypy/changeset/c932756506d4/

Log:hg merge default

diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py 
b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -273,7 +273,8 @@
 return r
 
 def contains(self, val):
-assert not isinstance(val, long)
+if not we_are_translated():
+assert not isinstance(val, long)
 if not isinstance(val, int):
 if ((not self.has_lower or self.lower == MININT) and
 not self.has_upper or self.upper == MAXINT):
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit