[pypy-commit] pypy vecopt: added ABC optimization that is turned on when executed when user code is vectorized
Author: Richard Plangger
Branch: vecopt
Changeset: r78558:04f489dd60dd
Date: 2015-07-16 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/04f489dd60dd/
Log:added ABC optimization that is turned on when executed when user
code is vectorized note that this optimization versions the loop
immediatly (to be tested) it introduces a guard before the loop is
entered to remove guards that are contained within the loop body
diff --git a/rpython/jit/metainterp/history.py
b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -746,7 +746,7 @@
class LoopVersion(object):
-def __init__(self, operations, opt_ops, invariant_arg_count=0,
aligned=False):
+def __init__(self, operations, aligned=False):
self.operations = operations
self.aligned = aligned
self.faildescrs = []
@@ -756,6 +756,8 @@
label = operations[idx]
self.label_pos = idx
self.inputargs = label.getarglist()
+
+def register_all_guards(self, opt_ops, invariant_arg_count=0):
idx = index_of_first(rop.LABEL, opt_ops)
assert idx >= 0
version_failargs = opt_ops[idx].getarglist()
@@ -769,6 +771,7 @@
for op in opt_ops:
if op.is_guard():
+import pdb; pdb.set_trace()
assert isinstance(op, GuardResOp)
descr = op.getdescr()
if descr.loop_version():
@@ -780,6 +783,13 @@
op.setfailargs(version_failargs)
op.rd_snapshot = None
+def register_guard(self, op):
+assert isinstance(op, GuardResOp)
+descr = op.getdescr()
+self.faildescrs.append(descr)
+op.setfailargs(self.inputargs)
+op.rd_snapshot = None
+
def copy_operations(self):
return [op.clone() for op in self.operations]
@@ -803,7 +813,6 @@
call_pure_results = None
logops = None
quasi_immutable_deps = None
-versions = None
def _token(*args):
raise Exception("TreeLoop.token is killed")
@@ -816,6 +825,7 @@
def __init__(self, name):
self.name = name
+self.versions = []
# self.operations = list of ResOperations
# ops of the kind 'guard_xxx' contain a further list of operations,
# which may itself contain 'guard_xxx' and so on, making a tree.
@@ -841,6 +851,14 @@
""" return the first operation having the same opnum or -1 """
return index_of_first(opnum, self.operations)
+def snapshot(self):
+version = LoopVersion(self.copy_operations(), [])
+self.versions.append(version)
+return version
+
+def copy_operations(self):
+return [ op.clone() for op in self.operations ]
+
def get_display_text(self):# for graphpage.py
return self.name + '\n' + repr(self.inputargs)
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py
b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -73,7 +73,7 @@
or warmstate.vectorize_user):
optimize_vector(metainterp_sd, jitdriver_sd, loop,
optimizations, inline_short_preamble,
-start_state, warmstate.vec_cost)
+start_state, warmstate)
else:
return optimize_unroll(metainterp_sd, jitdriver_sd, loop,
optimizations, inline_short_preamble,
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py
b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -891,8 +891,9 @@
return mycoeff + self.constant - (othercoeff + other.constant)
def emit_operations(self, opt, result_box=None):
-assert not self.is_identity()
box = self.var
+if self.is_identity():
+return box
last_op = None
if self.coefficient_mul != 1:
box_result = box.clonebox()
@@ -904,25 +905,31 @@
last_op = ResOperation(rop.INT_FLOORDIV, [box,
ConstInt(self.coefficient_div)], box_result)
opt.emit_operation(last_op)
box = box_result
-if self.constant != 0:
+if self.constant > 0:
box_result = box.clonebox()
last_op = ResOperation(rop.INT_ADD, [box,
ConstInt(self.constant)], box_result)
opt.emit_operation(last_op)
box = box_result
+if self.constant < 0:
+box_result = box.clonebox()
+last_op = ResOperation(rop.INT_SUB, [box,
ConstInt(self.constant)], box_result)
+opt.emit_operation(last_op)
+box = box_result
if result_box is not None:
[pypy-commit] pypy vecopt: rpython hints, saving the loop version the compileloopversion descr
Author: Richard Plangger
Branch: vecopt
Changeset: r78559:841f77615665
Date: 2015-07-16 16:06 +0200
http://bitbucket.org/pypy/pypy/changeset/841f77615665/
Log:rpython hints, saving the loop version the compileloopversion descr
diff --git a/rpython/jit/backend/x86/assembler.py
b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -612,7 +612,7 @@
addr = rawstart + tok.pos_jump_offset
tok.faildescr.adr_jump_offset = addr
descr = tok.faildescr
-if isinstance(descr, CompileLoopVersionDescr):
+if descr.loop_version():
continue # patch them later
relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4)
assert rx86.fits_in_32bits(relative_target)
diff --git a/rpython/jit/metainterp/compile.py
b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -725,6 +725,7 @@
operations = None
inputargs = None
faillocs = None
+version = None
def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
assert 0, "this guard must never fail"
diff --git a/rpython/jit/metainterp/history.py
b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -553,7 +553,7 @@
def save_to_descr(self, descr, position):
from rpython.jit.metainterp.compile import ResumeGuardDescr
from rpython.jit.metainterp.resume import AccumInfo
-assert isinstance(descr,ResumeGuardDescr)
+assert isinstance(descr, ResumeGuardDescr)
ai = AccumInfo(descr.rd_accum_list, position, self.operator, self.var)
descr.rd_accum_list = ai
@@ -758,6 +758,7 @@
self.inputargs = label.getarglist()
def register_all_guards(self, opt_ops, invariant_arg_count=0):
+from rpython.jit.metainterp.compile import CompileLoopVersionDescr
idx = index_of_first(rop.LABEL, opt_ops)
assert idx >= 0
version_failargs = opt_ops[idx].getarglist()
@@ -771,21 +772,26 @@
for op in opt_ops:
if op.is_guard():
-import pdb; pdb.set_trace()
assert isinstance(op, GuardResOp)
descr = op.getdescr()
if descr.loop_version():
-# currently there is only ONE versioning,
-# that is the original loop after unrolling.
-# if there are more possibilites, let the descr
-# know which loop version he preferes
-self.faildescrs.append(descr)
-op.setfailargs(version_failargs)
-op.rd_snapshot = None
+assert isinstance(descr, CompileLoopVersionDescr)
+if descr.version is None:
+# currently there is only ONE versioning,
+# that is the original loop after unrolling.
+# if there are more possibilites, let the descr
+# know which loop version he preferes
+descr.version = self
+self.faildescrs.append(descr)
+op.setfailargs(version_failargs)
+op.rd_snapshot = None
def register_guard(self, op):
+from rpython.jit.metainterp.compile import CompileLoopVersionDescr
assert isinstance(op, GuardResOp)
descr = op.getdescr()
+assert isinstance(descr, CompileLoopVersionDescr)
+descr.version = self
self.faildescrs.append(descr)
op.setfailargs(self.inputargs)
op.rd_snapshot = None
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py
b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -16,12 +16,12 @@
""" An object wrapper around a guard. Helps to determine
if one guard implies another
"""
+_attrs_ = ('index', 'op', 'cmp_op', 'rhs', 'lhs')
+
def __init__(self, index, op, cmp_op, index_vars):
self.index = index
self.op = op
self.cmp_op = cmp_op
-self.lhs_key = None
-self.rhs_key = None
lhs = cmp_op.getarg(0)
self.lhs = index_vars.get(lhs, None)
if self.lhs is None:
@@ -32,6 +32,15 @@
if self.rhs is None:
self.rhs = IndexVar(rhs)
+def setindex(self, index):
+self.index = index
+
+def setoperation(self, op):
+self.op = op
+
+def setcmp(self, c):
+self.cmp_op = c
+
def getleftkey(self):
return self.lhs.getvariable()
@@ -108,11 +117,10 @@
self.index = other.index
descr = myop.getdescr()
-if we_are_translated():
-descr.copy_all_attributes_from(other.o
[pypy-commit] pypy indexing: Kill Chunks and simply use a new_view() function instead
Author: Ronan Lamy Branch: indexing Changeset: r78560:360ba1502914 Date: 2015-07-16 18:47 +0100 http://bitbucket.org/pypy/pypy/changeset/360ba1502914/ Log:Kill Chunks and simply use a new_view() function instead diff --git a/pypy/module/micronumpy/arrayops.py b/pypy/module/micronumpy/arrayops.py --- a/pypy/module/micronumpy/arrayops.py +++ b/pypy/module/micronumpy/arrayops.py @@ -5,7 +5,7 @@ from pypy.module.micronumpy.base import convert_to_array, W_NDimArray from pypy.module.micronumpy.converters import clipmode_converter from pypy.module.micronumpy.strides import ( -Chunk, Chunks, shape_agreement, shape_agreement_multiple) +Chunk, new_view, shape_agreement, shape_agreement_multiple) from .casting import find_binop_result_dtype, find_result_type @@ -148,7 +148,8 @@ continue chunks[axis] = Chunk(axis_start, axis_start + arr.get_shape()[axis], 1, arr.get_shape()[axis]) -Chunks(chunks).apply(space, res).implementation.setslice(space, arr) +view = new_view(space, res, chunks) +view.implementation.setslice(space, arr) axis_start += arr.get_shape()[axis] return res @@ -162,9 +163,8 @@ shape = [arr.get_shape()[0] * repeats] w_res = W_NDimArray.from_shape(space, shape, arr.get_dtype(), w_instance=arr) for i in range(repeats): -chunks = Chunks([Chunk(i, shape[0] - repeats + i, repeats, - orig_size)]) -view = chunks.apply(space, w_res) +chunks = [Chunk(i, shape[0] - repeats + i, repeats, orig_size)] +view = new_view(space, w_res, chunks) view.implementation.setslice(space, arr) else: axis = space.int_w(w_axis) @@ -176,7 +176,7 @@ for i in range(repeats): chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats, orig_size) -view = Chunks(chunks).apply(space, w_res) +view = new_view(space, w_res, chunks) view.implementation.setslice(space, arr) return w_res diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -1,7 +1,7 @@ from pypy.interpreter.error import OperationError, oefmt from rpython.rlib import jit, rgc from rpython.rlib.buffer import Buffer -from rpython.rlib.debug import make_sure_not_resized, debug_print +from rpython.rlib.debug import make_sure_not_resized from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \ raw_storage_getitem, raw_storage_setitem, RAW_STORAGE from rpython.rtyper.lltypesystem import rffi, lltype, llmemory @@ -10,13 +10,11 @@ ArrayArgumentException, W_NumpyObject from pypy.module.micronumpy.iterators import ArrayIter from pypy.module.micronumpy.strides import ( -Chunk, Chunks, NewAxisChunk, EllipsisChunk, +Chunk, new_view, NewAxisChunk, EllipsisChunk, calc_strides, calc_new_strides, shape_agreement, calculate_broadcast_strides, calc_backstrides, calc_start, is_c_contiguous, is_f_contiguous) from rpython.rlib.objectmodel import keepalive_until_here -from rpython.rtyper.annlowlevel import cast_gcref_to_instance -from pypy.interpreter.baseobjspace import W_Root class BaseConcreteArray(object): _immutable_fields_ = ['dtype?', 'storage', 'start', 'size', 'shape[*]', @@ -225,16 +223,16 @@ space.isinstance_w(w_idx, space.w_slice)): if len(self.get_shape()) == 0: raise oefmt(space.w_ValueError, "cannot slice a 0-d array") -return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]) +return [Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))] elif isinstance(w_idx, W_NDimArray) and w_idx.is_scalar(): w_idx = w_idx.get_scalar_value().item(space) if not space.isinstance_w(w_idx, space.w_int) and \ not space.isinstance_w(w_idx, space.w_bool): raise OperationError(space.w_IndexError, space.wrap( "arrays used as indices must be of integer (or boolean) type")) -return Chunks([Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))]) +return [Chunk(*space.decode_index4(w_idx, self.get_shape()[0]))] elif space.is_w(w_idx, space.w_None): -return Chunks([NewAxisChunk()]) +return [NewAxisChunk()] result = [] i = 0 has_ellipsis = False @@ -253,7 +251,7 @@ result.append(Chunk(*space.decode_index4(w_item, self.get_shape()[i]))) i += 1 -return Chunks(result) +return result def descr_getitem(self, space, orig_arr, w_index): try: @@ -262,7 +260,7 @@ except IndexError: # not a single result
[pypy-commit] pypy py3k: hg merge default
Author: Manuel Jacob
Branch: py3k
Changeset: r78561:05c251509e69
Date: 2015-07-17 01:58 +0200
http://bitbucket.org/pypy/pypy/changeset/05c251509e69/
Log:hg merge default
diff too long, truncating to 2000 out of 4707 lines
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.1.2
+Version: 1.2.0
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI, CDefError, FFIError
from .ffiplatform import VerificationError, VerificationMissing
-__version__ = "1.1.2"
-__version_info__ = (1, 1, 2)
+__version__ = "1.2.0"
+__version_info__ = (1, 2, 0)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -236,6 +236,30 @@
cdecl = self._typeof(cdecl)
return self._backend.newp(cdecl, init)
+def new_allocator(self, alloc=None, free=None,
+ should_clear_after_alloc=True):
+"""Return a new allocator, i.e. a function that behaves like ffi.new()
+but uses the provided low-level 'alloc' and 'free' functions.
+
+'alloc' is called with the size as argument. If it returns NULL, a
+MemoryError is raised. 'free' is called with the result of 'alloc'
+as argument. Both can be either Python function or directly C
+functions. If 'free' is None, then no free function is called.
+If both 'alloc' and 'free' are None, the default is used.
+
+If 'should_clear_after_alloc' is set to False, then the memory
+returned by 'alloc' is assumed to be already cleared (or you are
+fine with garbage); otherwise CFFI will clear it.
+"""
+compiled_ffi = self._backend.FFI()
+allocator = compiled_ffi.new_allocator(alloc, free,
+ should_clear_after_alloc)
+def allocate(cdecl, init=None):
+if isinstance(cdecl, basestring):
+cdecl = self._typeof(cdecl)
+return allocator(cdecl, init)
+return allocate
+
def cast(self, cdecl, source):
"""Similar to a C cast: returns an instance of the named C
type initialized with the given 'source'. The source is
@@ -286,7 +310,7 @@
"""
return self._backend.from_buffer(self.BCharA, python_buffer)
-def callback(self, cdecl, python_callable=None, error=None):
+def callback(self, cdecl, python_callable=None, error=None, onerror=None):
"""Return a callback object or a decorator making such a
callback object. 'cdecl' must name a C function pointer type.
The callback invokes the specified 'python_callable' (which may
@@ -298,7 +322,8 @@
if not callable(python_callable):
raise TypeError("the 'python_callable' argument "
"is not callable")
-return self._backend.callback(cdecl, python_callable, error)
+return self._backend.callback(cdecl, python_callable,
+ error, onerror)
if isinstance(cdecl, basestring):
cdecl = self._typeof(cdecl, consider_function_as_funcptr=True)
if python_callable is None:
@@ -327,6 +352,13 @@
data. Later, when this new cdata object is garbage-collected,
'destructor(old_cdata_object)' will be called.
"""
+try:
+gcp = self._backend.gcp
+except AttributeError:
+pass
+else:
+return gcp(cdata, destructor)
+#
with self._lock:
try:
gc_weakrefs = self.gc_weakrefs
@@ -428,6 +460,8 @@
raise TypeError("ffi.include() expects an argument that is also of"
" type cffi.FFI, not %r" % (
type(ffi_to_include).__name__,))
+if ffi_to_include is self:
+raise ValueError("self.include(self)")
with ffi_to_include._lock:
with self._lock:
self._parser.include(ffi_to_include._parser)
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -989,7 +989,8 @@
def cast(self, BType, source):
return BType._cast_from(source)
-def callback(self, BType, source, error):
+def callback(self, BType, source, error, onerror):
+assert onerror i
[pypy-commit] pypy py3k: Mark this as an implementation detail.
Author: Manuel Jacob Branch: py3k Changeset: r78565:b8ef0188e57e Date: 2015-07-17 04:45 +0200 http://bitbucket.org/pypy/pypy/changeset/b8ef0188e57e/ Log:Mark this as an implementation detail. diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py --- a/lib-python/3/test/test_dis.py +++ b/lib-python/3/test/test_dis.py @@ -1,6 +1,6 @@ # Minimal tests for dis module -from test.support import run_unittest, captured_stdout +from test.support import run_unittest, captured_stdout, check_impl_detail import unittest import sys import dis @@ -209,7 +209,8 @@ def test_disassemble_str(self): self.do_disassembly_test(expr_str, dis_expr_str) self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str) -self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str) +if check_impl_detail(): +self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str) code_info_code_info = """\ Name: code_info ___ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
[pypy-commit] pypy py3k: 2to3
Author: Manuel Jacob Branch: py3k Changeset: r78563:e3d0cafafa1a Date: 2015-07-17 04:21 +0200 http://bitbucket.org/pypy/pypy/changeset/e3d0cafafa1a/ Log:2to3 diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py --- a/pypy/interpreter/test/test_zzpickle_and_slow.py +++ b/pypy/interpreter/test/test_zzpickle_and_slow.py @@ -537,9 +537,9 @@ yield 0 x = f() -x.next() +next(x) try: -x.next() +next(x) except StopIteration: y = pickle.loads(pickle.dumps(x)) assert 'finished' in y.__name__ ___ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
[pypy-commit] pypy py3k: 2to3
Author: Manuel Jacob
Branch: py3k
Changeset: r78564:6da82c1b733b
Date: 2015-07-17 04:23 +0200
http://bitbucket.org/pypy/pypy/changeset/6da82c1b733b/
Log:2to3
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py
b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -300,7 +300,7 @@
seen = []
def myalloc(size):
seen.append(size)
-return ffi.new("char[]", "X" * size)
+return ffi.new("char[]", b"X" * size)
def myfree(raw):
seen.append(raw)
alloc1 = ffi.new_allocator(myalloc, myfree)
@@ -333,7 +333,7 @@
seen = []
def myalloc(size):
seen.append(size)
-return ffi.new("char[]", "X" * size)
+return ffi.new("char[]", b"X" * size)
alloc1 = ffi.new_allocator(myalloc)# no 'free'
p1 = alloc1("int[10]")
assert seen == [40]
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
[pypy-commit] pypy py3k: Remove forgotten merge conflict marker and fix failing tests.
Author: Manuel Jacob
Branch: py3k
Changeset: r78562:34a546ca0da4
Date: 2015-07-17 04:20 +0200
http://bitbucket.org/pypy/pypy/changeset/34a546ca0da4/
Log:Remove forgotten merge conflict marker and fix failing tests.
diff --git a/pypy/module/imp/test/test_import.py
b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -769,30 +769,29 @@
def test_import_star_finds_submodules_with___all__(self):
for case in ["not-imported-yet", "already-imported"]:
d = {}
-exec "from pkg.withall import *" in d
+exec("from pkg.withall import *", d)
assert d["foobar"].found == 123
def test_import_star_does_not_find_submodules_without___all__(self):
for case in ["not-imported-yet", "already-imported"]:
d = {}
-exec "from pkg.withoutall import *" in d
+exec("from pkg.withoutall import *", d)
assert "foobar" not in d
import pkg.withoutall.foobar # <- import it here only
for case in ["not-imported-yet", "already-imported"]:
d = {}
-exec "from pkg.withoutall import *" in d
+exec("from pkg.withoutall import *", d)
assert d["foobar"].found == 123
def test_import_star_with_bogus___all__(self):
for case in ["not-imported-yet", "already-imported"]:
try:
-exec "from pkg.bogusall import *" in {}
+exec("from pkg.bogusall import *", {})
except TypeError:
pass# 'int' object does not support indexing
else:
raise AssertionError("should have failed")
-<<< local
def test_source_encoding(self):
import imp
import encoded
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
[pypy-commit] pypy dtypes-compatability: fix failing tests, even more logic for creating repr(dtype)
Author: mattip
Branch: dtypes-compatability
Changeset: r78568:1b6a885833e3
Date: 2015-07-17 01:46 +0300
http://bitbucket.org/pypy/pypy/changeset/1b6a885833e3/
Log:fix failing tests, even more logic for creating repr(dtype)
diff --git a/pypy/module/micronumpy/descriptor.py
b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -224,7 +224,7 @@
if not self.is_record():
return space.newlist([space.newtuple([space.wrap(""),
self.descr_get_str(space,
simple=simple)])])
-elif (self.alignment > 1 and style != 'descr') or force_dict:
+elif (self.alignment > 1 and not style.startswith('descr')) or
force_dict:
# we need to force a sorting order for the keys,
# so return a string instead of a dict. Also, numpy formats
# the lists without spaces between elements, so we cannot simply
@@ -241,9 +241,13 @@
offset, subdtype = self.fields[name]
if subdtype.is_record():
substr = [space.str_w(space.str(subdtype.descr_get_descr(
-space, style='substr'))), ","]
+space,
style='descr_subdtype'))), ","]
elif subdtype.subdtype is not None:
-substr = ["'", subdtype.subdtype.get_str(ignore=''), "',"]
+substr = ["(", space.str_w(space.str(
+subdtype.subdtype.descr_get_descr(space,
style='descr_subdtype'))),
+', ',
+space.str_w(space.repr(space.newtuple([space.wrap(s)
for s in subdtype.shape]))),
+"),"]
else:
substr = ["'", subdtype.get_str(ignore=''), "',"]
formats += substr
@@ -272,7 +276,7 @@
offsets[-1] = offsets[-1][:-1] + ']'
names[-1] = names[-1][:-1] + ']'
titles[-1] = titles[-1][:-1] + ']'
-if self.alignment < 2:
+if self.alignment < 2 or style.endswith('subdtype'):
suffix = "}"
elif style == 'str':
suffix = ", 'aligned':True}"
@@ -292,9 +296,9 @@
for name, title in self.names:
offset, subdtype = self.fields[name]
show_offsets = False
-if total != offset:
+if total != offset and len(subdtype.shape) < 1:
# whoops, need to use other format
-return self.descr_get_descr(space, style=style,
force_dict=True)
+return self.descr_get_descr(space, style=style +
'_subdtype', force_dict=True)
total += subdtype.elsize
ignore = '|'
if title:
@@ -311,7 +315,7 @@
if subdtype.shape != []:
subdescr.append(subdtype.descr_get_shape(space))
descr.append(space.newtuple(subdescr[:]))
-if self.alignment >= 0:
+if self.alignment >= 0 and not style.endswith('subdtype'):
return
space.wrap(space.str_w(space.repr(space.newlist(descr))) + ', align=True')
return space.newlist(descr)
@@ -711,8 +715,6 @@
assert isinstance(subdtype, W_Dtype)
if alignment >= 0:
maxalign = max(subdtype.alignment, maxalign)
-if not subdtype.is_record():
-maxalign = max(subdtype.elsize, maxalign)
delta = subdtype.alignment
# Set offset to the next power-of-two above delta
delta = (delta + maxalign -1) & (-maxalign)
@@ -937,6 +939,9 @@
shape_w = space.fixedview(w_shape)
if len(shape_w) < 1:
return None
+elif len(shape_w) == 1 and space.isinstance_w(shape_w[0], space.w_tuple):
+# (base_dtype, new_dtype) dtype spectification
+return None
shape = []
for w_dim in shape_w:
try:
diff --git a/pypy/module/micronumpy/test/test_dtypes.py
b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1417,7 +1417,7 @@
('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
('bright', '>f4', (8, 36))])],
align=True)
-assert_equal(str(dt),
+assert str(dt) == (
"{'names':['top','bottom'], "
"'formats':[([('tiles', ('>f4', (64, 64)), (1,)), "
"('rtile', '>f4', (64, 36))], (3,)),"
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
[pypy-commit] pypy default: whoops
Author: mattip Branch: Changeset: r78573:a1f4fa173a0d Date: 2015-07-17 02:00 +0300 http://bitbucket.org/pypy/pypy/changeset/a1f4fa173a0d/ Log:whoops diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -38,7 +38,7 @@ Partial implementation of unicode dtype and unicode scalars. -.. branch: dtypes-compatibility +.. branch: dtypes-compatability Improve compatibility with numpy dtypes; handle offsets to create unions, fix str() and repr(), allow specifying itemsize, metadata and titles, add flags, ___ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
[pypy-commit] pypy default: merge dtypes-compatability, which almost finishes dtype attributes, creation, and display
Author: mattip
Branch:
Changeset: r78572:c4992c458ed4
Date: 2015-07-17 01:52 +0300
http://bitbucket.org/pypy/pypy/changeset/c4992c458ed4/
Log:merge dtypes-compatability, which almost finishes dtype attributes,
creation, and display
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -37,3 +37,9 @@
.. branch: unicode-dtype
Partial implementation of unicode dtype and unicode scalars.
+
+.. branch: dtypes-compatibility
+
+Improve compatibility with numpy dtypes; handle offsets to create unions,
+fix str() and repr(), allow specifying itemsize, metadata and titles, add
flags,
+allow subclassing dtype
diff --git a/pypy/module/micronumpy/appbridge.py
b/pypy/module/micronumpy/appbridge.py
--- a/pypy/module/micronumpy/appbridge.py
+++ b/pypy/module/micronumpy/appbridge.py
@@ -8,6 +8,7 @@
w__commastring = None
w_array_repr = None
w_array_str = None
+w__usefields = None
def __init__(self, space):
pass
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -563,7 +563,7 @@
elif space.isinstance_w(w_item, space.w_int):
indx = space.int_w(w_item)
try:
-item = self.dtype.names[indx]
+item = self.dtype.names[indx][0]
except IndexError:
if indx < 0:
indx += len(self.dtype.names)
@@ -596,7 +596,7 @@
try:
ofs, dtype = self.dtype.fields[item]
except KeyError:
-raise oefmt(space.w_IndexError, "222only integers, slices (`:`), "
+raise oefmt(space.w_IndexError, "only integers, slices (`:`), "
"ellipsis (`...`), numpy.newaxis (`None`) and integer or "
"boolean arrays are valid indices")
dtype.store(self.arr, self.ofs, ofs,
diff --git a/pypy/module/micronumpy/compile.py
b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -47,6 +47,9 @@
def lookup(self, name):
return self.getdictvalue(self, name)
+def getname(self, space):
+return self.name
+
class FakeSpace(ObjSpace):
w_ValueError = W_TypeObject("ValueError")
w_TypeError = W_TypeObject("TypeError")
diff --git a/pypy/module/micronumpy/constants.py
b/pypy/module/micronumpy/constants.py
--- a/pypy/module/micronumpy/constants.py
+++ b/pypy/module/micronumpy/constants.py
@@ -92,6 +92,21 @@
ARRAY_ELEMENTSTRIDES = 0x0080 # strides are units of the dtype element size
ARRAY_NOTSWAPPED = 0x0200 #native byte order
+#dtype flags
+ITEM_REFCOUNT = 0x01
+ITEM_HASOBJECT = 0x01
+LIST_PICKLE = 0x02
+ITEM_IS_POINTER = 0x04
+NEEDS_INIT = 0x08
+NEEDS_PYAPI = 0x10
+USE_GETITEM = 0x20
+USE_SETITEM = 0x40
+ALIGNED_STRUCT = 0x80
+FROM_FIELDS = NEEDS_INIT | LIST_PICKLE | ITEM_REFCOUNT | NEEDS_PYAPI
+OBJECT_DTYPE_FLAGS = (LIST_PICKLE | USE_GETITEM | ITEM_IS_POINTER |
+ITEM_REFCOUNT | NEEDS_INIT | NEEDS_PYAPI)
+
+
LITTLE = '<'
BIG = '>'
NATIVE = '='
diff --git a/pypy/module/micronumpy/descriptor.py
b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -10,6 +10,7 @@
from rpython.rlib.objectmodel import (
specialize, compute_hash, we_are_translated, enforceargs)
from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
+from rpython.rlib.rstring import StringBuilder
from pypy.module.micronumpy import types, boxes, support, constants as NPY
from .base import W_NDimArray
from pypy.module.micronumpy.appbridge import get_appbridge_cache
@@ -54,11 +55,11 @@
class W_Dtype(W_Root):
_immutable_fields_ = [
"itemtype?", "w_box_type", "byteorder?", "names?", "fields?",
-"elsize?", "alignment?", "shape?", "subdtype?", "base?"]
+"elsize?", "alignment?", "shape?", "subdtype?", "base?", "flags?"]
@enforceargs(byteorder=SomeChar())
def __init__(self, itemtype, w_box_type, byteorder=NPY.NATIVE, names=[],
- fields={}, elsize=None, shape=[], subdtype=None):
+ fields={}, elsize=-1, shape=[], subdtype=None):
self.itemtype = itemtype
self.w_box_type = w_box_type
if itemtype.get_element_size() == 1 or isinstance(itemtype,
types.ObjectType):
@@ -66,16 +67,21 @@
self.byteorder = byteorder
self.names = names
self.fields = fields
-if elsize is None:
+if elsize < 0:
elsize = itemtype.get_element_size()
self.elsize = elsize
-self.alignment = itemtype.alignment
self.shape = shape
self.subdtype = subdtype
+self.flags = 0
+self.metadata = None
+if isinstance(itemtype, types.ObjectType):
+self.flags = NPY
[pypy-commit] pypy dtypes-compatability: more failing tests
Author: mattip
Branch: dtypes-compatability
Changeset: r78566:b5da748a0419
Date: 2015-07-15 23:42 +0300
http://bitbucket.org/pypy/pypy/changeset/b5da748a0419/
Log:more failing tests
diff --git a/pypy/module/micronumpy/test/test_dtypes.py
b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -490,6 +490,8 @@
assert False,'self._ptr_size unknown'
# Issue gh-2798
if '__pypy__' in sys.builtin_module_names:
+a = np.array(['a'], dtype="O")
+raises(NotImplementedError, a.astype, ("O", [("name", "O")]))
skip("(base_dtype, new_dtype) dtype specification discouraged")
a = np.array(['a'], dtype="O").astype(("O", [("name", "O")]))
assert a[0] == 'a'
@@ -1407,6 +1409,24 @@
"('bottom', [('bleft', ('>f4', (8, 64)), (1,)), "
"('bright', '>f4', (8, 36))])])")
+# If the sticky aligned flag is set to True, it makes the
+# str() function use a dict representation with an 'aligned' flag
+dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)),
+('rtile', '>f4', (64, 36))],
+(3,)),
+ ('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
+ ('bright', '>f4', (8, 36))])],
+ align=True)
+assert_equal(str(dt),
+"{'names':['top','bottom'], "
+ "'formats':[([('tiles', ('>f4', (64, 64)), (1,)), "
+ "('rtile', '>f4', (64, 36))], (3,)),"
+ "[('bleft', ('>f4', (8, 64)), (1,)), "
+ "('bright', '>f4', (8, 36))]], "
+ "'offsets':[0,76800], "
+ "'itemsize':8, "
+ "'aligned':True}")
+
dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'],
'offsets': [0, 1, 2],
'titles': ['Red pixel', 'Green pixel', 'Blue pixel']},
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
[pypy-commit] pypy dtypes-compatability: merge default into branch
Author: mattip
Branch: dtypes-compatability
Changeset: r78569:191ef045b0b6
Date: 2015-07-17 01:46 +0300
http://bitbucket.org/pypy/pypy/changeset/191ef045b0b6/
Log:merge default into branch
diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h
--- a/pypy/module/_vmprof/src/getpc.h
+++ b/pypy/module/_vmprof/src/getpc.h
@@ -132,7 +132,7 @@
}
};
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
// See comment above struct CallUnrollInfo. Only try instruction
// flow matching if both eip and esp looks reasonable.
const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
@@ -168,7 +168,7 @@
typedef int ucontext_t;
#endif
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
return NULL;
}
@@ -178,7 +178,7 @@
// the right value for your system, and add it to the list in
// configure.ac (or set it manually in your config.h).
#else
-inline void* GetPC(ucontext_t *signal_ucontext) {
+void* GetPC(ucontext_t *signal_ucontext) {
return (void*)signal_ucontext->PC_FROM_UCONTEXT; // defined in config.h
}
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -262,13 +262,31 @@
int marker = MARKER_TRAILER;
write(profile_file, &marker, 1);
+#ifdef __linux__
// copy /proc/PID/maps to the end of the profile file
sprintf(buf, "/proc/%d/maps", getpid());
-src = fopen(buf, "r");
+src = fopen(buf, "r");
+if (!src) {
+vmprof_error = "error opening proc maps";
+return -1;
+}
while ((size = fread(buf, 1, BUFSIZ, src))) {
write(profile_file, buf, size);
}
fclose(src);
+#else
+// freebsd and mac
+sprintf(buf, "procstat -v %d", getpid());
+src = popen(buf, "r");
+if (!src) {
+vmprof_error = "error calling procstat";
+return -1;
+}
+while ((size = fread(buf, 1, BUFSIZ, src))) {
+write(profile_file, buf, size);
+}
+pclose(src);
+#endif
close(profile_file);
return 0;
}
diff --git a/pypy/objspace/std/test/test_typeobject.py
b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1031,6 +1031,48 @@
A.__dict__['x'] = 5
assert A.x == 5
+def test_we_already_got_one_1(self):
+# Issue #2079: highly obscure: CPython complains if we say
+# ``__slots__="__dict__"`` and there is already a __dict__...
+# but from the "best base" only. If the __dict__ comes from
+# another base, it doesn't complain. Shrug and copy the logic.
+class A(object):
+__slots__ = ()
+class B(object):
+pass
+class C(A, B): # "best base" is A
+__slots__ = ("__dict__",)
+class D(A, B): # "best base" is A
+__slots__ = ("__weakref__",)
+try:
+class E(B, A): # "best base" is B
+__slots__ = ("__dict__",)
+except TypeError, e:
+assert 'we already got one' in str(e)
+else:
+raise AssertionError("TypeError not raised")
+try:
+class F(B, A): # "best base" is B
+__slots__ = ("__weakref__",)
+except TypeError, e:
+assert 'we already got one' in str(e)
+else:
+raise AssertionError("TypeError not raised")
+
+def test_we_already_got_one_2(self):
+class A(object):
+__slots__ = ()
+class B:
+pass
+class C(A, B): # "best base" is A
+__slots__ = ("__dict__",)
+class D(A, B): # "best base" is A
+__slots__ = ("__weakref__",)
+class C(B, A): # "best base" is A
+__slots__ = ("__dict__",)
+class D(B, A): # "best base" is A
+__slots__ = ("__weakref__",)
+
class AppTestWithMethodCacheCounter:
spaceconfig = {"objspace.std.withmethodcachecounter": True}
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -1022,7 +1022,7 @@
w_self.nslots = w_bestbase.nslots
return hasoldstylebase
-def create_all_slots(w_self, hasoldstylebase):
+def create_all_slots(w_self, hasoldstylebase, w_bestbase):
space = w_self.space
dict_w = w_self.dict_w
if '__slots__' not in dict_w:
@@ -1040,12 +1040,12 @@
for w_slot_name in slot_names_w:
slot_name = space.str_w(w_slot_name)
if slot_name == '__dict__':
-if wantdict or w_self.hasdict:
+if wantdict or w_bestbase.hasdict:
raise oefmt(space.w_TypeError,
[pypy-commit] pypy dtypes-compatability: document branch to be merged
Author: mattip Branch: dtypes-compatability Changeset: r78570:c386a754e4f4 Date: 2015-07-17 01:50 +0300 http://bitbucket.org/pypy/pypy/changeset/c386a754e4f4/ Log:document branch to be merged diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -37,3 +37,9 @@ .. branch: unicode-dtype Partial implementation of unicode dtype and unicode scalars. + +.. branch: dtypes-compatibility + +Improve compatibility with numpy dtypes; handle offsets to create unions, +fix str() and repr(), allow specifying itemsize, metadata and titles, add flags, +allow subclassing dtype ___ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
[pypy-commit] pypy dtypes-compatability: close branch
Author: mattip Branch: dtypes-compatability Changeset: r78571:29f6c9e399e1 Date: 2015-07-17 01:51 +0300 http://bitbucket.org/pypy/pypy/changeset/29f6c9e399e1/ Log:close branch ___ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
[pypy-commit] pypy dtypes-compatability: use a single ''.join() instead of lots of string concatenation
Author: mattip
Branch: dtypes-compatability
Changeset: r78567:9692f8a54feb
Date: 2015-07-16 21:30 +0300
http://bitbucket.org/pypy/pypy/changeset/9692f8a54feb/
Log:use a single ''.join() instead of lots of string concatenation
diff --git a/pypy/module/micronumpy/descriptor.py
b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -10,6 +10,7 @@
from rpython.rlib.objectmodel import (
specialize, compute_hash, we_are_translated, enforceargs)
from rpython.rlib.rarithmetic import r_longlong, r_ulonglong
+from rpython.rlib.rstring import StringBuilder
from pypy.module.micronumpy import types, boxes, support, constants as NPY
from .base import W_NDimArray
from pypy.module.micronumpy.appbridge import get_appbridge_cache
@@ -228,10 +229,10 @@
# so return a string instead of a dict. Also, numpy formats
# the lists without spaces between elements, so we cannot simply
# do str(names)
-names = "'names':["
-formats = "'formats':["
-offsets = "'offsets':["
-titles = "'titles':["
+names = ["'names':["]
+formats = ["'formats':["]
+offsets = ["'offsets':["]
+titles = ["'titles':["]
use_titles = False
show_offsets = False
offsets_n = []
@@ -239,16 +240,16 @@
for name, title in self.names:
offset, subdtype = self.fields[name]
if subdtype.is_record():
-substr = space.str_w(space.str(subdtype.descr_get_descr(
-space, style='substr'))) + ","
+substr = [space.str_w(space.str(subdtype.descr_get_descr(
+space, style='substr'))), ","]
elif subdtype.subdtype is not None:
-substr = "'" + subdtype.subdtype.get_str(ignore='') + "',"
+substr = ["'", subdtype.subdtype.get_str(ignore=''), "',"]
else:
-substr = "'" + subdtype.get_str(ignore='') + "',"
+substr = ["'", subdtype.get_str(ignore=''), "',"]
formats += substr
-offsets += str(offset) + ','
-names += "'" + name + "',"
-titles += "'" + str(title) + "',"
+offsets += [str(offset), ',']
+names += ["'", name, "',"]
+titles += ["'", str(title), "',"]
if title is not None:
use_titles = True
if total != offset:
@@ -264,10 +265,13 @@
for i in range(len(offsets_n)):
if offsets_n[i] != self.alignment * i:
show_offsets = True
-formats = formats[:-1] + ']'
-offsets = offsets[:-1] + ']'
-names = names[:-1] + ']'
-titles = titles[:-1] + ']'
+if use_titles and not show_offsets:
+return self.descr_get_descr(space, style='descr')
+# replace the last , with a ]
+formats[-1] = formats[-1][:-1] + ']'
+offsets[-1] = offsets[-1][:-1] + ']'
+names[-1] = names[-1][:-1] + ']'
+titles[-1] = titles[-1][:-1] + ']'
if self.alignment < 2:
suffix = "}"
elif style == 'str':
@@ -276,17 +280,12 @@
suffix = '}'
else:
suffix = "}, align=True"
-if use_titles and not show_offsets:
-return self.descr_get_descr(space, style='descr')
-elif use_titles:
-return space.wrap('{' + names + ', ' + formats + ', ' +
-offsets + ', ' + titles + ", 'itemsize':" +
-str(self.elsize) + suffix)
-else:
-return space.wrap('{' + names + ', ' + formats + ', ' +
-offsets + ', ' + "'itemsize':" + str(self.elsize) +
-suffix)
-
+s_as_list = ['{'] + names + [', '] + formats + [', '] + offsets +
[', ']
+if use_titles:
+s_as_list += titles + [', ']
+
+s_as_list += ["'itemsize':", str(self.elsize), suffix]
+return space.wrap(''.join(s_as_list))
else:
descr = []
total = 0
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
