Author: Armin Rigo <[email protected]>
Branch: stmgc-c8
Changeset: r75738:053bac719b8c
Date: 2015-02-06 17:29 +0100
http://bitbucket.org/pypy/pypy/changeset/053bac719b8c/
Log: hg merge stmgc-c7, and import stmgc/c8
diff too long, truncating to 2000 out of 4037 lines
diff --git a/lib_pypy/atomic.py b/lib_pypy/atomic.py
deleted file mode 100644
--- a/lib_pypy/atomic.py
+++ /dev/null
@@ -1,24 +0,0 @@
-"""
-API for accessing the multithreading extensions of PyPy
-"""
-import thread
-
-try:
- from __pypy__ import thread as _thread
- from __pypy__.thread import (atomic, getsegmentlimit,
- hint_commit_soon, is_atomic)
-except ImportError:
- # Not a STM-enabled PyPy. We can still provide a version of 'atomic'
- # that is good enough for our purposes. With this limited version,
- # an atomic block in thread X will not prevent running thread Y, if
- # thread Y is not within an atomic block at all.
- atomic = thread.allocate_lock()
-
- def getsegmentlimit():
- return 1
-
- def hint_commit_soon():
- pass
-
- def is_atomic():
- return atomic.locked()
diff --git a/lib_pypy/pypy_test/test_transaction.py
b/lib_pypy/pypy_test/test_transaction.py
--- a/lib_pypy/pypy_test/test_transaction.py
+++ b/lib_pypy/pypy_test/test_transaction.py
@@ -1,3 +1,4 @@
+import py
from lib_pypy import transaction
N = 1000
@@ -7,9 +8,9 @@
def test_simple_random_order():
for x in range(N):
lst = []
- for i in range(10):
- transaction.add(lst.append, i)
- transaction.run()
+ with transaction.TransactionQueue():
+ for i in range(10):
+ transaction.add(lst.append, i)
if VERBOSE:
print lst
assert sorted(lst) == range(10), lst
@@ -22,8 +23,8 @@
i += 1
if i < 10:
transaction.add(do_stuff, i)
- transaction.add(do_stuff, 0)
- transaction.run()
+ with transaction.TransactionQueue():
+ transaction.add(do_stuff, 0)
if VERBOSE:
print lst
assert lst == range(10), lst
@@ -36,9 +37,9 @@
j += 1
if j < 10:
transaction.add(do_stuff, i, j)
- for i in range(5):
- transaction.add(do_stuff, i, 0)
- transaction.run()
+ with transaction.TransactionQueue():
+ for i in range(5):
+ transaction.add(do_stuff, i, 0)
if VERBOSE:
print lsts
assert lsts == (range(10),) * 5, lsts
@@ -56,10 +57,10 @@
else:
lsts[i].append('foo')
raise FooError
- for i in range(10):
- transaction.add(do_stuff, i, 0)
try:
- transaction.run()
+ with transaction.TransactionQueue():
+ for i in range(10):
+ transaction.add(do_stuff, i, 0)
except FooError:
pass
else:
@@ -77,8 +78,9 @@
def test_number_of_transactions_reported():
- transaction.add(lambda: None)
- transaction.run()
+ py.test.skip("not reimplemented")
+ with transaction.TransactionQueue():
+ transaction.add(lambda: None)
assert transaction.number_of_transactions_in_last_run() == 1
def add_transactions(l):
@@ -86,11 +88,52 @@
for x in range(l[0]):
transaction.add(add_transactions, l[1:])
- transaction.add(add_transactions, [10, 10, 10])
- transaction.run()
+ with transaction.TransactionQueue():
+ transaction.add(add_transactions, [10, 10, 10])
assert transaction.number_of_transactions_in_last_run() == 1111
+def test_stmidset():
+ s = transaction.stmidset()
+ key1 = []
+ key2 = []
+ s.add(key1)
+ assert key1 in s
+ assert key2 not in s
+ s.add(key2)
+ assert key1 in s
+ assert key2 in s
+ s.remove(key1)
+ assert key1 not in s
+ assert key2 in s
+ py.test.raises(KeyError, s.remove, key1)
+ s.discard(key1)
+ assert key1 not in s
+ assert key2 in s
+ s.discard(key2)
+ assert key2 not in s
+
+def test_stmiddict():
+ d = transaction.stmiddict()
+ key1 = []
+ key2 = []
+ py.test.raises(KeyError, "d[key1]")
+ d[key1] = 5
+ assert d[key1] == 5
+ assert key1 in d
+ assert d.get(key1) == 5
+ assert d.get(key1, 42) == 5
+ del d[key1]
+ py.test.raises(KeyError, "d[key1]")
+ assert key1 not in d
+ assert d.get(key1) is None
+ assert d.get(key1, 42) == 42
+ assert d.setdefault(key1, 42) == 42
+ assert d.setdefault(key1, 43) == 42
+ assert d.setdefault(key2) is None
+ assert d[key2] is None
+
+
def run_tests():
for name in sorted(globals().keys()):
if name.startswith('test_'):
diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -6,16 +6,16 @@
give a simple-to-use API.
Note that some rough edges still need to be sorted out; for now you
-have to explicitly set the number of threads to use by calling
-set_num_threads(), or you get a default of 4.
-
+have to explicitly set the number of threads to use by passing the
+'nb_segments' argument to TransactionQueue(), or you get a default of 4
+(or whatever the compiled-in maximum is).
"""
from __future__ import with_statement
import sys, thread, collections, cStringIO, linecache
try:
- from __pypy__.thread import atomic, is_atomic
+ from pypystm import atomic, is_atomic
except ImportError:
# Not a STM-enabled PyPy. We can use a regular lock for 'atomic',
# which is good enough for our purposes. With this limited version,
@@ -37,63 +37,117 @@
signals_enabled = _SignalsEnabled()
try:
- from __pypy__.thread import last_abort_info
+ from pypystm import hint_commit_soon
except ImportError:
# Not a STM-enabled PyPy.
- def last_abort_info():
+ def hint_commit_soon():
return None
+try:
+ from pypystm import getsegmentlimit
+except ImportError:
+ # Not a STM-enabled PyPy.
+ def getsegmentlimit():
+ return 1
-def set_num_threads(num):
- """Set the number of threads to use."""
- if num < 1:
- raise ValueError("'num' must be at least 1, got %r" % (num,))
- if _thread_pool.in_transaction:
- raise TransactionError("cannot change the number of threads "
- "while running transactions")
- _thread_pool.num_threads = num
+try:
+ from pypystm import hashtable
+except ImportError:
+ # Not a STM-enabled PyPy.
+ hashtable = dict
+
+class stmidset(object):
+ def __init__(self):
+ self._hashtable = hashtable()
+
+ def add(self, key):
+ self._hashtable[id(key)] = key
+
+ def __contains__(self, key):
+ return id(key) in self._hashtable
+
+ def remove(self, key):
+ del self._hashtable[id(key)]
+
+ def discard(self, key):
+ try:
+ del self._hashtable[id(key)]
+ except KeyError:
+ pass
+
+class stmiddict(object):
+ def __init__(self):
+ self._hashtable = hashtable()
+
+ def __getitem__(self, key):
+ return self._hashtable[id(key)][1]
+
+ def __setitem__(self, key, value):
+ self._hashtable[id(key)] = (key, value)
+
+ def __delitem__(self, key):
+ del self._hashtable[id(key)]
+
+ def __contains__(self, key):
+ return id(key) in self._hashtable
+
+ def get(self, key, default=None):
+ try:
+ return self._hashtable[id(key)][1]
+ except KeyError:
+ return default
+
+ def setdefault(self, key, default=None):
+ return self._hashtable.setdefault(id(key), (key, default))[1]
+
+
+# ------------------------------------------------------------
class TransactionError(Exception):
pass
-# XXX right now uses the same API as the old pypy-stm. This will
-# be redesigned later.
-
def add(f, *args, **kwds):
- """Register the call 'f(*args, **kwds)' as running a new
- transaction. If we are currently running in a transaction too, the
- new transaction will only start after the end of the current
- transaction. Note that if the current transaction or another running
- in the meantime raises an exception, all pending transactions are
- cancelled.
+ """Register a new transaction that will be done by 'f(*args, **kwds)'.
+ Must be called within the transaction in the "with TransactionQueue()"
+ block, or within a transaction started by this one, directly or
+ indirectly.
"""
_thread_local.pending.append((f, args, kwds))
-def run():
- """Run the pending transactions, as well as all transactions started
- by them, and so on. The order is random and undeterministic. Must
- be called from the main program, i.e. not from within another
- transaction. If at some point all transactions are done, returns.
- If a transaction raises an exception, it propagates here; in this
- case all pending transactions are cancelled.
+class TransactionQueue(object):
+ """Use in 'with TransactionQueue():'. Creates a queue of
+ transactions. The first transaction in the queue is the content of
+ the 'with:' block, which is immediately started.
+
+ Any transaction can register new transactions that will be run
+ after the current one is finished, using the global function add().
"""
- tpool = _thread_pool
- if tpool.in_transaction:
- raise TransactionError("recursive invocation of transaction.run()")
- if not _thread_local.pending:
- return # nothing to do
- try:
- tpool.setup()
- tpool.run()
- finally:
- tpool.teardown()
- tpool.reraise()
-def number_of_transactions_in_last_run():
- return _thread_pool.transactions_run
+ def __init__(self, nb_segments=0):
+ if nb_segments <= 0:
+ nb_segments = getsegmentlimit()
+ _thread_pool.ensure_threads(nb_segments)
+
+ def __enter__(self):
+ if hasattr(_thread_local, "pending"):
+ raise TransactionError(
+ "recursive invocation of TransactionQueue()")
+ if is_atomic():
+ raise TransactionError(
+ "invocation of TransactionQueue() from an atomic context")
+ _thread_local.pending = []
+ atomic.__enter__()
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ atomic.__exit__(exc_type, exc_value, traceback)
+ pending = _thread_local.pending
+ del _thread_local.pending
+ if exc_type is None and len(pending) > 0:
+ _thread_pool.run(pending)
+
# ____________________________________________________________
@@ -101,159 +155,118 @@
class _ThreadPool(object):
def __init__(self):
- try:
- from __pypy__.thread import getsegmentlimit
- self.num_threads = getsegmentlimit()
- except ImportError:
- self.num_threads = 4
- self.in_transaction = False
- self.transactions_run = None
+ self.lock_running = thread.allocate_lock()
+ self.lock_done_running = thread.allocate_lock()
+ self.lock_done_running.acquire()
+ self.nb_threads = 0
+ self.deque = collections.deque()
+ self.locks = []
+ self.lock_deque = thread.allocate_lock()
+ self.exception = []
- def setup(self):
- # a mutex to protect parts of _grab_next_thing_to_do()
- self.lock_mutex = thread.allocate_lock()
- # this lock is released if and only if there are things to do in
- # 'self.pending'; both are modified together, with the lock_mutex.
- self.lock_pending = thread.allocate_lock()
- # this lock is released when we are finished at the end
- self.lock_if_released_then_finished = thread.allocate_lock()
- self.lock_if_released_then_finished.acquire()
+ def ensure_threads(self, n):
+ if n > self.nb_threads:
+ with self.lock_running:
+ for i in range(self.nb_threads, n):
+ assert len(self.locks) == self.nb_threads
+ self.nb_threads += 1
+ thread.start_new_thread(self.thread_runner, ())
+ # The newly started thread should run immediately into
+ # the case 'if len(self.locks) == self.nb_threads:'
+ # and release this lock. Wait until it does.
+ self.lock_done_running.acquire()
+
+ def run(self, pending):
+ # For now, can't run multiple threads with each an independent
+ # TransactionQueue(): they are serialized.
+ with self.lock_running:
+ assert self.exception == []
+ assert len(self.deque) == 0
+ deque = self.deque
+ with self.lock_deque:
+ deque.extend(pending)
+ try:
+ for i in range(len(pending)):
+ self.locks.pop().release()
+ except IndexError: # pop from empty list
+ pass
+ #
+ self.lock_done_running.acquire()
+ #
+ if self.exception:
+ exc_type, exc_value, exc_traceback = self.exception
+ del self.exception[:]
+ raise exc_type, exc_value, exc_traceback
+
+ def thread_runner(self):
+ deque = self.deque
+ lock = thread.allocate_lock()
+ lock.acquire()
+ pending = []
+ _thread_local.pending = pending
+ lock_deque = self.lock_deque
+ exception = self.exception
#
- self.pending = _thread_local.pending
- # there must be pending items at the beginning, which means that
- # 'lock_pending' can indeed be released
- assert self.pending
- _thread_local.pending = None
- #
- self.num_waiting_threads = 0
- self.transactions_run = 0
- self.finished = False
- self.got_exception = []
- self.in_transaction = True
+ while True:
+ #
+ # Look at the deque and try to fetch the next item on the left.
+ # If empty, we add our lock to the 'locks' list.
+ lock_deque.acquire()
+ if deque:
+ next_transaction = deque.popleft()
+ lock_deque.release()
+ else:
+ self.locks.append(lock)
+ if len(self.locks) == self.nb_threads:
+ self.lock_done_running.release()
+ lock_deque.release()
+ #
+ # Now wait until our lock is released.
+ lock.acquire()
+ continue
+ #
+ # Now we have a next_transaction. Run it.
+ assert len(pending) == 0
+ while True:
+ f, args, kwds = next_transaction
+ with atomic:
+ if len(exception) == 0:
+ try:
+ f(*args, **kwds)
+ except:
+ exception.extend(sys.exc_info())
+ del next_transaction
+ #
+ # If no new 'pending' transactions have been added, exit
+ # this loop and go back to fetch more from the deque.
+ if len(pending) == 0:
+ break
+ #
+ # If we have some new 'pending' transactions, add them
+ # to the right of the deque and pop the next one from
+ # the left. As we do this atomically with the
+ # 'lock_deque', we are sure that the deque cannot be
+ # empty before the popleft(). (We do that even when
+ # 'len(pending) == 1' instead of simply assigning the
+ # single item to 'next_transaction', because it looks
+ # like a good idea to preserve some first-in-first-out
+ # approximation.)
+ with self.lock_deque:
+ deque.extend(pending)
+ next_transaction = deque.popleft()
+ try:
+ for i in range(1, len(pending)):
+ self.locks.pop().release()
+ except IndexError: # pop from empty list
+ pass
+ del pending[:]
- def run(self):
- # start the N threads
- task_counters = [[0] for i in range(self.num_threads)]
- for counter in task_counters:
- thread.start_new_thread(self._run_thread, (counter,))
- # now wait. When we manage to acquire the following lock, then
- # we are finished.
- self.lock_if_released_then_finished.acquire()
- self.transactions_run = sum(x[0] for x in task_counters)
-
- def teardown(self):
- self.in_transaction = False
- self.pending = None
- self.lock_if_released_then_finished = None
- self.lock_pending = None
- self.lock_mutex = None
- _thread_local.pending = collections.deque()
-
- def reraise(self):
- exc = self.got_exception
- self.got_exception = None
- if exc:
- raise exc[0], exc[1], exc[2] # exception, value, traceback
-
- def _run_thread(self, counter):
- tloc_pending = _thread_local.pending
- got_exception = self.got_exception
- try:
- while True:
- self._do_it(self._grab_next_thing_to_do(tloc_pending),
- got_exception)
- counter[0] += 1
- except _Done:
- pass
-
- def _grab_next_thing_to_do(self, tloc_pending):
- if tloc_pending:
- # grab the next thing to do from the thread-local deque
- next = tloc_pending.popleft()
- # add the rest, if any, to the global 'pending'
- if tloc_pending:
- #
- self.lock_mutex.acquire()
- if not self.pending:
- # self.pending is empty so far, but we are adding stuff.
- # we have to release the following lock.
- self.lock_pending.release()
- self.pending.extend(tloc_pending)
- self.lock_mutex.release()
- #
- tloc_pending.clear()
- return next
- #
- self.lock_mutex.acquire()
- while True:
- try:
- next = self.pending.popleft()
- except IndexError:
- # self.pending is empty: wait until it no longer is.
- pass
- else:
- # self.pending was not empty. If now it is empty, then
- # fix the status of 'lock_pending'.
- if not self.pending:
- self.lock_pending.acquire()
- self.lock_mutex.release()
- return next
- #
- # first check if all N threads are waiting here.
- assert not self.finished
- self.num_waiting_threads += 1
- if self.num_waiting_threads == self.num_threads:
- # yes, so finished! unlock this to wake up the other
- # threads, which are all waiting on the following acquire().
- self.finished = True
- self.lock_pending.release()
- #
- self.lock_mutex.release()
- self.lock_pending.acquire()
- self.lock_pending.release()
- self.lock_mutex.acquire()
- #
- self.num_waiting_threads -= 1
- if self.finished:
- last_one_to_leave = self.num_waiting_threads == 0
- self.lock_mutex.release()
- if last_one_to_leave:
- self.lock_if_released_then_finished.release()
- raise _Done
-
- @staticmethod
- def _do_it((f, args, kwds), got_exception):
- # this is a staticmethod in order to make sure that we don't
- # accidentally use 'self' in the atomic block.
- try:
- while True:
- with signals_enabled:
- with atomic:
- info = last_abort_info()
- if info is None:
- if not got_exception:
- f(*args, **kwds)
- # else return early if already an exc to reraise
- return
- report_abort_info(info)
- except:
- got_exception[:] = sys.exc_info()
_thread_pool = _ThreadPool()
+_thread_local = thread._local()
-class _Done(Exception):
- pass
-
-
-class _ThreadLocal(thread._local):
- def __init__(self):
- self.pending = collections.deque()
-
-_thread_local = _ThreadLocal()
-
-
-def report_abort_info(info):
+def XXXreport_abort_info(info):
header = info[0]
f = cStringIO.StringIO()
if len(info) > 1:
@@ -279,3 +292,27 @@
header[1], 'atom '*header[3], 'inev '*(header[4]>1),
header[5], header[6])
sys.stderr.write(f.getvalue())
+
+
+class threadlocalproperty(object):
+ def __init__(self, *default):
+ self.tl_default = default
+ self.tl_name = intern(str(id(self)))
+
+ def tl_get(self, obj):
+ try:
+ return obj._threadlocalproperties
+ except AttributeError:
+ return obj.__dict__.setdefault('_threadlocalproperties',
+ thread._local())
+
+ def __get__(self, obj, cls=None):
+ if obj is None:
+ return self
+ return getattr(self.tl_get(obj), self.tl_name, *self.tl_default)
+
+ def __set__(self, obj, value):
+ setattr(self.tl_get(obj), self.tl_name, value)
+
+ def __delete__(self, obj):
+ delattr(self.tl_get(obj), self.tl_name)
diff --git a/pypy/goal/targetpypystandalone.py
b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -226,9 +226,10 @@
# expose the following variables to ease debugging
global space, entry_point
- if config.translation.stm:
+ if config.translation.stm or config.objspace.usemodules.pypystm:
+ config.translation.stm = True
config.translation.thread = True
- config.objspace.usemodules._stm = True
+ config.objspace.usemodules.pypystm = True
if config.objspace.allworkingmodules:
from pypy.config.pypyoption import enable_allworkingmodules
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1022,6 +1022,9 @@
def newlist_unicode(self, list_u):
return self.newlist([self.wrap(u) for u in list_u])
+ def newlist_int(self, list_i):
+ return self.newlist([self.wrap(i) for i in list_i])
+
def newlist_hint(self, sizehint):
from pypy.objspace.std.listobject import make_empty_list_with_size
return make_empty_list_with_size(self, sizehint)
diff --git a/pypy/interpreter/executioncontext.py
b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -34,7 +34,7 @@
self.w_profilefuncarg = None
#
if self.space.config.translation.stm:
- from pypy.module._stm.ec import initialize_execution_context
+ from pypy.module.pypystm.ec import initialize_execution_context
initialize_execution_context(self)
self.thread_disappeared = False # might be set to True after
os.fork()
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -29,22 +29,11 @@
class ThreadModule(MixedModule):
appleveldefs = {
'signals_enabled': 'app_signal.signals_enabled',
- 'atomic': 'app_atomic.atomic',
- 'exclusive_atomic': 'app_atomic.exclusive_atomic',
}
interpleveldefs = {
'_signals_enter': 'interp_signal.signals_enter',
'_signals_exit': 'interp_signal.signals_exit',
- '_atomic_enter': 'interp_atomic.atomic_enter',
- '_exclusive_atomic_enter': 'interp_atomic.exclusive_atomic_enter',
- '_atomic_exit': 'interp_atomic.atomic_exit',
- 'getsegmentlimit': 'interp_atomic.getsegmentlimit',
- 'hint_commit_soon': 'interp_atomic.hint_commit_soon',
- 'is_atomic': 'interp_atomic.is_atomic',
- 'error': 'space.fromcache(pypy.module.thread.error.Cache).w_error',
}
- def activate(self, space):
- return self.space.config.objspace.usemodules.thread
class IntOpModule(MixedModule):
diff --git a/pypy/module/__pypy__/test/test_signal.py
b/pypy/module/__pypy__/test/test_signal.py
--- a/pypy/module/__pypy__/test/test_signal.py
+++ b/pypy/module/__pypy__/test/test_signal.py
@@ -4,7 +4,7 @@
class AppTestMinimal:
- spaceconfig = dict(usemodules=['__pypy__'])
+ spaceconfig = dict(usemodules=['__pypy__', 'thread'])
def test_signal(self):
from __pypy__ import thread
diff --git a/pypy/module/_stm/__init__.py b/pypy/module/pypystm/__init__.py
rename from pypy/module/_stm/__init__.py
rename to pypy/module/pypystm/__init__.py
--- a/pypy/module/_stm/__init__.py
+++ b/pypy/module/pypystm/__init__.py
@@ -4,12 +4,24 @@
class Module(MixedModule):
appleveldefs = {
+ 'atomic': 'app_atomic.atomic',
+ 'exclusive_atomic': 'app_atomic.exclusive_atomic',
}
interpleveldefs = {
+ '_atomic_enter': 'interp_atomic.atomic_enter',
+ '_exclusive_atomic_enter': 'interp_atomic.exclusive_atomic_enter',
+ '_atomic_exit': 'interp_atomic.atomic_exit',
+ 'getsegmentlimit': 'interp_atomic.getsegmentlimit',
+ 'hint_commit_soon': 'interp_atomic.hint_commit_soon',
+ 'is_atomic': 'interp_atomic.is_atomic',
+ 'error': 'space.fromcache(pypy.module.thread.error.Cache).w_error',
+
'local': 'local.STMLocal',
'count': 'count.count',
'hashtable': 'hashtable.W_Hashtable',
'time': 'time.time',
'clock': 'time.clock',
+ 'stmset': 'stmset.W_STMSet',
+ 'stmdict': 'stmdict.W_STMDict',
}
diff --git a/pypy/module/__pypy__/app_atomic.py
b/pypy/module/pypystm/app_atomic.py
rename from pypy/module/__pypy__/app_atomic.py
rename to pypy/module/pypystm/app_atomic.py
--- a/pypy/module/__pypy__/app_atomic.py
+++ b/pypy/module/pypystm/app_atomic.py
@@ -1,12 +1,12 @@
-from __pypy__ import thread
+import pypystm
class Atomic(object):
- __enter__ = thread._atomic_enter
- __exit__ = thread._atomic_exit
+ __enter__ = pypystm._atomic_enter
+ __exit__ = pypystm._atomic_exit
class ExclusiveAtomic(object):
- __enter__ = thread._exclusive_atomic_enter
- __exit__ = thread._atomic_exit
+ __enter__ = pypystm._exclusive_atomic_enter
+ __exit__ = pypystm._atomic_exit
atomic = Atomic()
exclusive_atomic = ExclusiveAtomic()
diff --git a/pypy/module/_stm/count.py b/pypy/module/pypystm/count.py
rename from pypy/module/_stm/count.py
rename to pypy/module/pypystm/count.py
--- a/pypy/module/_stm/count.py
+++ b/pypy/module/pypystm/count.py
@@ -1,5 +1,5 @@
"""
-_stm.count()
+pypystm.count()
"""
from rpython.rlib import rstm
diff --git a/pypy/module/_stm/ec.py b/pypy/module/pypystm/ec.py
rename from pypy/module/_stm/ec.py
rename to pypy/module/pypystm/ec.py
--- a/pypy/module/_stm/ec.py
+++ b/pypy/module/pypystm/ec.py
@@ -20,7 +20,7 @@
"""Called from ExecutionContext.__init__()."""
if ec.space.config.translation.rweakref:
from rpython.rlib import rweakref
- from pypy.module._stm.local import STMLocal
+ from pypy.module.pypystm.local import STMLocal
ec._thread_local_dicts = rweakref.RWeakKeyDictionary(STMLocal, W_Root)
else:
ec._thread_local_dicts = FakeWeakKeyDictionary()
diff --git a/pypy/module/_stm/hashtable.py b/pypy/module/pypystm/hashtable.py
rename from pypy/module/_stm/hashtable.py
rename to pypy/module/pypystm/hashtable.py
--- a/pypy/module/_stm/hashtable.py
+++ b/pypy/module/pypystm/hashtable.py
@@ -1,12 +1,13 @@
"""
-The class _stm.hashtable, mapping integers to objects.
+The class pypystm.hashtable, mapping integers to objects.
"""
from pypy.interpreter.baseobjspace import W_Root
from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
from rpython.rlib import rstm
+from rpython.rlib.rarithmetic import intmask
from rpython.rtyper.annlowlevel import cast_gcref_to_instance
from rpython.rtyper.annlowlevel import cast_instance_to_gcref
@@ -25,21 +26,68 @@
@unwrap_spec(key=int)
def setitem_w(self, key, w_value):
- gcref = cast_instance_to_gcref(w_value)
- self.h.set(key, gcref)
+ entry = self.h.lookup(key)
+ entry.object = cast_instance_to_gcref(w_value)
@unwrap_spec(key=int)
def delitem_w(self, space, key):
- gcref = self.h.get(key)
- if not gcref:
+ entry = self.h.lookup(key)
+ if not entry.object:
space.raise_key_error(space.wrap(key))
- self.h.set(key, rstm.NULL_GCREF)
+ entry.object = rstm.NULL_GCREF
@unwrap_spec(key=int)
def contains_w(self, space, key):
gcref = self.h.get(key)
return space.newbool(not not gcref)
+ @unwrap_spec(key=int, w_default=WrappedDefault(None))
+ def get_w(self, space, key, w_default):
+ gcref = self.h.get(key)
+ if not gcref:
+ return w_default
+ return cast_gcref_to_instance(W_Root, gcref)
+
+ @unwrap_spec(key=int, w_default=WrappedDefault(None))
+ def setdefault_w(self, space, key, w_default):
+ entry = self.h.lookup(key)
+ gcref = entry.object
+ if not gcref:
+ entry.object = cast_instance_to_gcref(w_default)
+ return w_default
+ return cast_gcref_to_instance(W_Root, gcref)
+
+ def len_w(self, space):
+ return space.wrap(self.h.len())
+
+ def keys_w(self, space):
+ array, count = self.h.list()
+ try:
+ lst = [intmask(array[i].index) for i in range(count)]
+ finally:
+ self.h.freelist(array)
+ return space.newlist_int(lst)
+
+ def values_w(self, space):
+ array, count = self.h.list()
+ try:
+ lst_w = [cast_gcref_to_instance(W_Root, array[i].object)
+ for i in range(count)]
+ finally:
+ self.h.freelist(array)
+ return space.newlist(lst_w)
+
+ def items_w(self, space):
+ array, count = self.h.list()
+ try:
+ lst_w = [space.newtuple([
+ space.wrap(intmask(array[i].index)),
+ cast_gcref_to_instance(W_Root, array[i].object)])
+ for i in range(count)]
+ finally:
+ self.h.freelist(array)
+ return space.newlist(lst_w)
+
def W_Hashtable___new__(space, w_subtype):
r = space.allocate_instance(W_Hashtable, w_subtype)
@@ -47,10 +95,17 @@
return space.wrap(r)
W_Hashtable.typedef = TypeDef(
- '_stm.hashtable',
+ 'pypystm.hashtable',
__new__ = interp2app(W_Hashtable___new__),
__getitem__ = interp2app(W_Hashtable.getitem_w),
__setitem__ = interp2app(W_Hashtable.setitem_w),
__delitem__ = interp2app(W_Hashtable.delitem_w),
__contains__ = interp2app(W_Hashtable.contains_w),
- )
+ get = interp2app(W_Hashtable.get_w),
+ setdefault = interp2app(W_Hashtable.setdefault_w),
+
+ __len__ = interp2app(W_Hashtable.len_w),
+ keys = interp2app(W_Hashtable.keys_w),
+ values = interp2app(W_Hashtable.values_w),
+ items = interp2app(W_Hashtable.items_w),
+)
diff --git a/pypy/module/__pypy__/interp_atomic.py
b/pypy/module/pypystm/interp_atomic.py
rename from pypy/module/__pypy__/interp_atomic.py
rename to pypy/module/pypystm/interp_atomic.py
diff --git a/pypy/module/_stm/local.py b/pypy/module/pypystm/local.py
rename from pypy/module/_stm/local.py
rename to pypy/module/pypystm/local.py
--- a/pypy/module/_stm/local.py
+++ b/pypy/module/pypystm/local.py
@@ -1,5 +1,5 @@
"""
-The '_stm.local' class, used for 'thread._local' with STM.
+The 'pypystm.local' class, used for 'thread._local' with STM.
"""
from pypy.interpreter.gateway import W_Root, interp2app
@@ -10,7 +10,7 @@
def _fill_untranslated(ec):
if not we_are_translated() and not hasattr(ec, '_thread_local_dicts'):
- from pypy.module._stm.ec import initialize_execution_context
+ from pypy.module.pypystm.ec import initialize_execution_context
initialize_execution_context(ec)
@@ -67,7 +67,7 @@
# No arguments allowed
pass
-STMLocal.typedef = TypeDef("_stm.local",
+STMLocal.typedef = TypeDef("pypystm.local",
__doc__ = "Thread-local data",
__new__ = interp2app(STMLocal.descr_local__new__.im_func),
__init__ = interp2app(STMLocal.descr_local__init__),
diff --git a/pypy/module/pypystm/stmdict.py b/pypy/module/pypystm/stmdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/stmdict.py
@@ -0,0 +1,243 @@
+"""
+The class pypystm.stmdict, giving a part of the regular 'dict' interface
+"""
+
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+
+from rpython.rlib import rstm, jit, rgc
+from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref
+from rpython.rtyper.lltypesystem import lltype, llmemory
+
+ARRAY = lltype.GcArray(llmemory.GCREF)
+PARRAY = lltype.Ptr(ARRAY)
+
+
+def find_equal_item(space, array, w_key):
+ w_item = cast_gcref_to_instance(W_Root, array[0])
+ if space.eq_w(w_key, w_item):
+ return 0
+ if len(array) > 2:
+ return _run_next_iterations(space, array, w_key)
+ return -1
+
[email protected]_look_inside
+def _run_next_iterations(space, array, w_key):
+ i = 2
+ limit = len(array)
+ while True:
+ w_item = cast_gcref_to_instance(W_Root, array[i])
+ if space.eq_w(w_key, w_item):
+ return i
+ i += 2
+ if i >= limit:
+ return -1
+
+def ll_arraycopy(source, dest, source_start, dest_start, length):
+ if we_are_translated():
+ rgc.ll_arraycopy(source, dest, source_start, dest_start, length)
+ else:
+ for i in range(length):
+ dest[dest_start + i] = source[source_start + i]
+
+def pop_from_entry(entry, space, w_key):
+ array = lltype.cast_opaque_ptr(PARRAY, entry.object)
+ if not array:
+ return None
+ i = find_equal_item(space, array, w_key)
+ if i < 0:
+ return None
+ # found
+ w_value = cast_gcref_to_instance(W_Root, array[i + 1])
+ L = len(array) - 2
+ if L == 0:
+ narray = lltype.nullptr(ARRAY)
+ else:
+ narray = lltype.malloc(ARRAY, L)
+ ll_arraycopy(array, narray, 0, 0, i)
+ ll_arraycopy(array, narray, i + 2, i, L - i)
+ entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+ return w_value
+
+
+class W_STMDict(W_Root):
+
+ def __init__(self):
+ self.h = rstm.create_hashtable()
+
+ def getitem_w(self, space, w_key):
+ hkey = space.hash_w(w_key)
+ gcref = self.h.get(hkey)
+ array = lltype.cast_opaque_ptr(PARRAY, gcref)
+ if array:
+ i = find_equal_item(space, array, w_key)
+ if i >= 0:
+ return cast_gcref_to_instance(W_Root, array[i + 1])
+ space.raise_key_error(w_key)
+
+ def setitem_w(self, space, w_key, w_value):
+ hkey = space.hash_w(w_key)
+ entry = self.h.lookup(hkey)
+ array = lltype.cast_opaque_ptr(PARRAY, entry.object)
+ if array:
+ i = find_equal_item(space, array, w_key)
+ if i >= 0:
+ # already there, update the value
+ array[i + 1] = cast_instance_to_gcref(w_value)
+ return
+ L = len(array)
+ narray = lltype.malloc(ARRAY, L + 2)
+ ll_arraycopy(array, narray, 0, 0, L)
+ else:
+ narray = lltype.malloc(ARRAY, 2)
+ L = 0
+ narray[L] = cast_instance_to_gcref(w_key)
+ narray[L + 1] = cast_instance_to_gcref(w_value)
+ entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+
+ def delitem_w(self, space, w_key):
+ hkey = space.hash_w(w_key)
+ entry = self.h.lookup(hkey)
+ if pop_from_entry(entry, space, w_key) is None:
+ space.raise_key_error(w_key)
+
+ def contains_w(self, space, w_key):
+ hkey = space.hash_w(w_key)
+ gcref = self.h.get(hkey)
+ array = lltype.cast_opaque_ptr(PARRAY, gcref)
+ if array and find_equal_item(space, array, w_key) >= 0:
+ return space.w_True
+ return space.w_False
+
+ @unwrap_spec(w_default=WrappedDefault(None))
+ def get_w(self, space, w_key, w_default):
+ hkey = space.hash_w(w_key)
+ gcref = self.h.get(hkey)
+ array = lltype.cast_opaque_ptr(PARRAY, gcref)
+ if array:
+ i = find_equal_item(space, array, w_key)
+ if i >= 0:
+ return cast_gcref_to_instance(W_Root, array[i + 1])
+ return w_default
+
+ def pop_w(self, space, w_key, w_default=None):
+ hkey = space.hash_w(w_key)
+ entry = self.h.lookup(hkey)
+ w_value = pop_from_entry(entry, space, w_key)
+ if w_value is not None:
+ return w_value
+ elif w_default is not None:
+ return w_default
+ else:
+ space.raise_key_error(w_key)
+
+ @unwrap_spec(w_default=WrappedDefault(None))
+ def setdefault_w(self, space, w_key, w_default):
+ hkey = space.hash_w(w_key)
+ entry = self.h.lookup(hkey)
+ array = lltype.cast_opaque_ptr(PARRAY, entry.object)
+ if array:
+ i = find_equal_item(space, array, w_key)
+ if i >= 0:
+ # already there, return the existing value
+ return cast_gcref_to_instance(W_Root, array[i + 1])
+ L = len(array)
+ narray = lltype.malloc(ARRAY, L + 2)
+ ll_arraycopy(array, narray, 0, 0, L)
+ else:
+ narray = lltype.malloc(ARRAY, 2)
+ L = 0
+ narray[L] = cast_instance_to_gcref(w_key)
+ narray[L + 1] = cast_instance_to_gcref(w_default)
+ entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+ return w_default
+
+ def get_length(self):
+ array, count = self.h.list()
+ try:
+ total_length_times_two = 0
+ for i in range(count):
+ subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+ assert subarray
+ total_length_times_two += len(subarray)
+ finally:
+ self.h.freelist(array)
+ return total_length_times_two >> 1
+
+ def get_keys_values_w(self, offset):
+ array, count = self.h.list()
+ try:
+ result_list_w = []
+ for i in range(count):
+ subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+ assert subarray
+ j = offset
+ limit = len(subarray)
+ while j < limit:
+ w_item = cast_gcref_to_instance(W_Root, subarray[j])
+ result_list_w.append(w_item)
+ j += 2
+ finally:
+ self.h.freelist(array)
+ return result_list_w
+
+ def get_items_w(self, space):
+ array, count = self.h.list()
+ try:
+ result_list_w = []
+ for i in range(count):
+ subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+ assert subarray
+ j = 0
+ limit = len(subarray)
+ while j < limit:
+ w_key = cast_gcref_to_instance(W_Root, subarray[j])
+ w_value = cast_gcref_to_instance(W_Root, subarray[j + 1])
+ result_list_w.append(space.newtuple([w_key, w_value]))
+ j += 2
+ finally:
+ self.h.freelist(array)
+ return result_list_w
+
+ def len_w(self, space):
+ return space.wrap(self.get_length())
+
+ def iter_w(self, space):
+ # not a real lazy iterator!
+ return space.iter(self.keys_w(space))
+
+ def keys_w(self, space):
+ return space.newlist(self.get_keys_values_w(offset=0))
+
+ def values_w(self, space):
+ return space.newlist(self.get_keys_values_w(offset=1))
+
+ def items_w(self, space):
+ return space.newlist(self.get_items_w(space))
+
+
+def W_STMDict___new__(space, w_subtype):
+ r = space.allocate_instance(W_STMDict, w_subtype)
+ r.__init__()
+ return space.wrap(r)
+
+W_STMDict.typedef = TypeDef(
+ 'pypystm.stmdict',
+ __new__ = interp2app(W_STMDict___new__),
+ __getitem__ = interp2app(W_STMDict.getitem_w),
+ __setitem__ = interp2app(W_STMDict.setitem_w),
+ __delitem__ = interp2app(W_STMDict.delitem_w),
+ __contains__ = interp2app(W_STMDict.contains_w),
+ get = interp2app(W_STMDict.get_w),
+ pop = interp2app(W_STMDict.pop_w),
+ setdefault = interp2app(W_STMDict.setdefault_w),
+
+ __len__ = interp2app(W_STMDict.len_w),
+ __iter__ = interp2app(W_STMDict.iter_w),
+ keys = interp2app(W_STMDict.keys_w),
+ values = interp2app(W_STMDict.values_w),
+ items = interp2app(W_STMDict.items_w),
+ )
diff --git a/pypy/module/pypystm/stmset.py b/pypy/module/pypystm/stmset.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/stmset.py
@@ -0,0 +1,146 @@
+"""
+The class pypystm.stmset, giving a part of the regular 'set' interface
+"""
+
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app
+
+from rpython.rlib import rstm, jit
+from rpython.rlib.rgc import ll_arraycopy
+from rpython.rlib.objectmodel import specialize
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref
+from rpython.rtyper.lltypesystem import lltype, llmemory
+
+ARRAY = lltype.GcArray(llmemory.GCREF)
+PARRAY = lltype.Ptr(ARRAY)
+
+
+def find_equal_item(space, array, w_key):
+ w_item = cast_gcref_to_instance(W_Root, array[0])
+ if space.eq_w(w_key, w_item):
+ return 0
+ if len(array) > 1:
+ return _run_next_iterations(space, array, w_key)
+ return -1
+
[email protected]_look_inside
+def _run_next_iterations(space, array, w_key):
+ i = 1
+ limit = len(array)
+ while True:
+ w_item = cast_gcref_to_instance(W_Root, array[i])
+ if space.eq_w(w_key, w_item):
+ return i
+ i += 1
+ if i >= limit:
+ return -1
+
+
+class W_STMSet(W_Root):
+
+ def __init__(self):
+ self.h = rstm.create_hashtable()
+
+ def contains_w(self, space, w_key):
+ hkey = space.hash_w(w_key)
+ gcref = self.h.get(hkey)
+ array = lltype.cast_opaque_ptr(PARRAY, gcref)
+ if array and find_equal_item(space, array, w_key) >= 0:
+ return space.w_True
+ return space.w_False
+
+ def add_w(self, space, w_key):
+ hkey = space.hash_w(w_key)
+ entry = self.h.lookup(hkey)
+ array = lltype.cast_opaque_ptr(PARRAY, entry.object)
+ if array:
+ if find_equal_item(space, array, w_key) >= 0:
+ return # already there
+ L = len(array)
+ narray = lltype.malloc(ARRAY, L + 1)
+ ll_arraycopy(array, narray, 0, 0, L)
+ else:
+ narray = lltype.malloc(ARRAY, 1)
+ L = 0
+ narray[L] = cast_instance_to_gcref(w_key)
+ entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+
+ def try_remove(self, space, w_key):
+ hkey = space.hash_w(w_key)
+ entry = self.h.lookup(hkey)
+ array = lltype.cast_opaque_ptr(PARRAY, entry.object)
+ if not array:
+ return False
+ i = find_equal_item(space, array, w_key)
+ if i < 0:
+ return False
+ # found
+ L = len(array) - 1
+ if L == 0:
+ narray = lltype.nullptr(ARRAY)
+ else:
+ narray = lltype.malloc(ARRAY, L)
+ ll_arraycopy(array, narray, 0, 0, i)
+ ll_arraycopy(array, narray, i + 1, i, L - i)
+ entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+ return True
+
+ def remove_w(self, space, w_key):
+ if not self.try_remove(space, w_key):
+ space.raise_key_error(w_key)
+
+ def discard_w(self, space, w_key):
+ self.try_remove(space, w_key)
+
+ def get_length(self):
+ array, count = self.h.list()
+ try:
+ total_length = 0
+ for i in range(count):
+ subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+ assert subarray
+ total_length += len(subarray)
+ finally:
+ self.h.freelist(array)
+ return total_length
+
+ def get_items_w(self):
+ array, count = self.h.list()
+ try:
+ result_list_w = []
+ for i in range(count):
+ subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+ assert subarray
+ for j in range(len(subarray)):
+ w_item = cast_gcref_to_instance(W_Root, subarray[j])
+ result_list_w.append(w_item)
+ finally:
+ self.h.freelist(array)
+ return result_list_w
+
+ def len_w(self, space):
+ return space.wrap(self.get_length())
+
+ def iter_w(self, space):
+ # not a real lazy iterator!
+ return space.iter(space.newlist(self.get_items_w()))
+
+
+def W_STMSet___new__(space, w_subtype):
+ r = space.allocate_instance(W_STMSet, w_subtype)
+ r.__init__()
+ return space.wrap(r)
+
+W_STMSet.typedef = TypeDef(
+ 'pypystm.stmset',
+ __new__ = interp2app(W_STMSet___new__),
+ __contains__ = interp2app(W_STMSet.contains_w),
+ add = interp2app(W_STMSet.add_w),
+ remove = interp2app(W_STMSet.remove_w),
+ discard = interp2app(W_STMSet.discard_w),
+
+ __len__ = interp2app(W_STMSet.len_w),
+ __iter__ = interp2app(W_STMSet.iter_w),
+ )
diff --git a/pypy/module/_stm/test/__init__.py
b/pypy/module/pypystm/test/__init__.py
rename from pypy/module/_stm/test/__init__.py
rename to pypy/module/pypystm/test/__init__.py
diff --git a/pypy/module/__pypy__/test/test_atomic.py
b/pypy/module/pypystm/test/test_atomic.py
rename from pypy/module/__pypy__/test/test_atomic.py
rename to pypy/module/pypystm/test/test_atomic.py
--- a/pypy/module/__pypy__/test/test_atomic.py
+++ b/pypy/module/pypystm/test/test_atomic.py
@@ -1,32 +1,13 @@
-from __future__ import with_statement
-from pypy.module.thread.test.support import GenericTestThread
-from rpython.rtyper.lltypesystem import rffi
-def test_bdecode(space):
- from pypy.module.__pypy__.interp_atomic import bdecode
- def bdec(s, expected):
- p = rffi.str2charp(s)
- w_obj, q = bdecode(space, p)
- assert q == rffi.ptradd(p, len(s))
- rffi.free_charp(p)
- w_expected = space.wrap(expected)
- assert space.eq_w(w_obj, w_expected)
-
- bdec("i123e", 123)
- bdec("i-123e", -123)
- bdec('12:+"*-%&/()=?\x00', '+"*-%&/()=?\x00')
- bdec("li123eli456eee", [123, [456]])
- bdec("l5:abcdei2ee", ["abcde", 2])
-
-
-class AppTestAtomic(GenericTestThread):
+class AppTestAtomic:
+ spaceconfig = dict(usemodules=['pypystm', 'thread'])
def test_simple(self):
- from __pypy__ import thread
- for atomic in thread.atomic, thread.exclusive_atomic:
+ import pypystm
+ for atomic in pypystm.atomic, pypystm.exclusive_atomic:
with atomic:
- assert thread.is_atomic()
+ assert pypystm.is_atomic()
try:
with atomic:
raise ValueError
@@ -34,40 +15,40 @@
pass
def test_nest_composable_atomic(self):
- from __pypy__ import thread
- with thread.atomic:
- with thread.atomic:
- assert thread.is_atomic()
- assert thread.is_atomic()
- assert not thread.is_atomic()
+ import pypystm
+ with pypystm.atomic:
+ with pypystm.atomic:
+ assert pypystm.is_atomic()
+ assert pypystm.is_atomic()
+ assert not pypystm.is_atomic()
def test_nest_composable_below_exclusive(self):
- from __pypy__ import thread
- with thread.exclusive_atomic:
- with thread.atomic:
- with thread.atomic:
- assert thread.is_atomic()
- assert thread.is_atomic()
- assert thread.is_atomic()
- assert not thread.is_atomic()
+ import pypystm
+ with pypystm.exclusive_atomic:
+ with pypystm.atomic:
+ with pypystm.atomic:
+ assert pypystm.is_atomic()
+ assert pypystm.is_atomic()
+ assert pypystm.is_atomic()
+ assert not pypystm.is_atomic()
def test_nest_exclusive_fails(self):
- from __pypy__ import thread
+ import pypystm
try:
- with thread.exclusive_atomic:
- with thread.exclusive_atomic:
- assert thread.is_atomic()
- except thread.error, e:
- assert not thread.is_atomic()
+ with pypystm.exclusive_atomic:
+ with pypystm.exclusive_atomic:
+ assert pypystm.is_atomic()
+ except pypystm.error, e:
+ assert not pypystm.is_atomic()
assert e.message == "exclusive_atomic block can't be entered
inside another atomic block"
def test_nest_exclusive_fails2(self):
- from __pypy__ import thread
+ import pypystm
try:
- with thread.atomic:
- with thread.exclusive_atomic:
- assert thread.is_atomic()
- assert thread.is_atomic()
- except thread.error, e:
- assert not thread.is_atomic()
+ with pypystm.atomic:
+ with pypystm.exclusive_atomic:
+ assert pypystm.is_atomic()
+ assert pypystm.is_atomic()
+ except pypystm.error, e:
+ assert not pypystm.is_atomic()
assert e.message == "exclusive_atomic block can't be entered
inside another atomic block"
diff --git a/pypy/module/_stm/test/test_count.py
b/pypy/module/pypystm/test/test_count.py
rename from pypy/module/_stm/test/test_count.py
rename to pypy/module/pypystm/test/test_count.py
--- a/pypy/module/_stm/test/test_count.py
+++ b/pypy/module/pypystm/test/test_count.py
@@ -1,10 +1,10 @@
class AppTestCount:
- spaceconfig = dict(usemodules=['_stm'])
+ spaceconfig = dict(usemodules=['pypystm'])
def test_count(self):
- import _stm
- x = _stm.count()
- y = _stm.count()
+ import pypystm
+ x = pypystm.count()
+ y = pypystm.count()
assert y == x + 1
diff --git a/pypy/module/_stm/test/test_hashtable.py
b/pypy/module/pypystm/test/test_hashtable.py
rename from pypy/module/_stm/test/test_hashtable.py
rename to pypy/module/pypystm/test/test_hashtable.py
--- a/pypy/module/_stm/test/test_hashtable.py
+++ b/pypy/module/pypystm/test/test_hashtable.py
@@ -1,11 +1,11 @@
class AppTestHashtable:
- spaceconfig = dict(usemodules=['_stm'])
+ spaceconfig = dict(usemodules=['pypystm'])
def test_simple(self):
- import _stm
- h = _stm.hashtable()
+ import pypystm
+ h = pypystm.hashtable()
h[42+65536] = "bar"
raises(KeyError, "h[42]")
h[42] = "foo"
@@ -16,3 +16,42 @@
raises(KeyError, "h[42]")
assert h[42+65536] == "bar"
raises(KeyError, "del h[42]")
+
+ def test_get_setdefault(self):
+ import pypystm
+ h = pypystm.hashtable()
+ assert h.get(42) is None
+ assert h.get(-43, None) is None
+ assert h.get(44, 81) == 81
+ raises(KeyError, "h[42]")
+ raises(KeyError, "h[-43]")
+ raises(KeyError, "h[44]")
+ assert h.setdefault(42) is None
+ assert h[42] is None
+ assert h.setdefault(42, "81") is None
+ assert h[42] is None
+ assert h.setdefault(44, "-81") == "-81"
+ assert h[44] == "-81"
+ assert h[42] is None
+
+ def test_len(self):
+ import pypystm
+ h = pypystm.hashtable()
+ assert len(h) == 0
+ h[42] = "foo"
+ assert len(h) == 1
+ h[43] = "bar"
+ assert len(h) == 2
+ h[42] = "baz"
+ assert len(h) == 2
+ del h[42]
+ assert len(h) == 1
+
+ def test_keys_values_items(self):
+ import pypystm
+ h = pypystm.hashtable()
+ h[42] = "foo"
+ h[43] = "bar"
+ assert sorted(h.keys()) == [42, 43]
+ assert sorted(h.values()) == ["bar", "foo"]
+ assert sorted(h.items()) == [(42, "foo"), (43, "bar")]
diff --git a/pypy/module/_stm/test/test_local.py
b/pypy/module/pypystm/test/test_local.py
rename from pypy/module/_stm/test/test_local.py
rename to pypy/module/pypystm/test/test_local.py
--- a/pypy/module/_stm/test/test_local.py
+++ b/pypy/module/pypystm/test/test_local.py
@@ -3,11 +3,11 @@
class AppTestSTMLocal(test_local.AppTestLocal):
spaceconfig = test_local.AppTestLocal.spaceconfig.copy()
- spaceconfig['usemodules'] += ('_stm',)
+ spaceconfig['usemodules'] += ('pypystm',)
def setup_class(cls):
test_local.AppTestLocal.setup_class.im_func(cls)
cls.w__local = cls.space.appexec([], """():
- import _stm
- return _stm.local
+ import pypystm
+ return pypystm.local
""")
diff --git a/pypy/module/pypystm/test/test_stmdict.py
b/pypy/module/pypystm/test/test_stmdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/test/test_stmdict.py
@@ -0,0 +1,118 @@
+
+
+class AppTestDict:
+ spaceconfig = dict(usemodules=['pypystm'])
+
+ def test_simple(self):
+ import pypystm
+ d = pypystm.stmdict()
+ raises(KeyError, "d[42.0]")
+ d[42.0] = "5"
+ assert d[42.0] == "5"
+ assert 42.0 in d
+ assert d.get(42.0) == "5"
+ assert d.get(42.0, 42) == "5"
+ del d[42.0]
+ raises(KeyError, "d[42.0]")
+ assert 42.0 not in d
+ assert d.get(42.0) is None
+ assert d.get(42.0, "41") == "41"
+ assert d.setdefault(42.0, "42") == "42"
+ assert d.setdefault(42.0, "43") == "42"
+ assert d.setdefault(42.5) is None
+ assert d[42.5] is None
+ assert d[42.0] == "42"
+ d[42.0] = "foo"
+ assert d[42.0] == "foo"
+
+ def test_hash_collision(self):
+ import pypystm
+ key1 = 5L
+ key2 = 5L + 2**64 - 1
+ key3 = 5L + 2**65 - 2
+ assert hash(key1) == hash(key2) == hash(key3)
+ d = pypystm.stmdict()
+ d[key1] = 1.0
+ d[key2] = 2.0
+ assert d[key1] == 1.0
+ assert key2 in d
+ assert key3 not in d
+ raises(KeyError, "d[key3]")
+ assert d.get(key3) is None
+ del d[key1]
+ assert key1 not in d
+ assert d[key2] == 2.0
+ assert key3 not in d
+ raises(KeyError, "del d[key1]")
+ del d[key2]
+ assert key1 not in d
+ assert key2 not in d
+ assert key3 not in d
+ raises(KeyError, "del d[key3]")
+ assert d.setdefault(key1, 5.0) == 5.0
+ assert d.setdefault(key2, 7.5) == 7.5
+ assert d.setdefault(key1, 2.3) == 5.0
+
+ def test_must_be_hashable(self):
+ import pypystm
+ d = pypystm.stmdict()
+ raises(TypeError, "d[[]]")
+ raises(TypeError, "d[[]] = 5")
+ raises(TypeError, "del d[[]]")
+ raises(TypeError, "[] in d")
+ raises(TypeError, "d.get([])")
+ raises(TypeError, "d.setdefault([], 0)")
+
+ def test_equal_elements(self):
+ import pypystm
+ d = pypystm.stmdict()
+ d[42.0] = "hello"
+ assert d[42] == "hello"
+ assert d.get(42L) == "hello"
+ assert d.get(42.001) is None
+
+ def test_list_from_dict(self):
+ import pypystm
+ d = pypystm.stmdict()
+ assert len(d) == 0
+ assert tuple(d) == ()
+ d[42.5] = "foo"
+ d[42.0] = ["bar"]
+ assert sorted(d) == [42.0, 42.5]
+ assert len(d) == 2
+ del d[42]
+ assert len(d) == 1
+ assert list(d) == [42.5]
+ #
+ class Key(object):
+ def __hash__(self):
+ return hash(42.5)
+ key3 = Key()
+ d[key3] = "other"
+ assert len(d) == 2
+ items = list(d)
+ assert items == [42.5, key3] or items == [key3, 42.5]
+
+ def test_keys_values_items(self):
+ import pypystm
+ d = pypystm.stmdict()
+ d[42.5] = "bar"
+ d[42.0] = "foo"
+ assert sorted(d.keys()) == [42.0, 42.5]
+ assert sorted(d.values()) == ["bar", "foo"]
+ assert sorted(d.items()) == [(42.0, "foo"), (42.5, "bar")]
+
+ def test_pop(self):
+ import pypystm
+ d = pypystm.stmdict()
+ raises(KeyError, d.pop, 42.0)
+ assert d.pop(42.0, "foo") == "foo"
+ raises(KeyError, "d[42.0]")
+ d[42.0] = "bar"
+ res = d.pop(42.0)
+ assert res == "bar"
+ raises(KeyError, "d[42.0]")
+ d[42.0] = "bar"
+ res = d.pop(42.0, "foo")
+ assert res == "bar"
+ raises(KeyError, "d[42.0]")
diff --git a/pypy/module/pypystm/test/test_stmset.py
b/pypy/module/pypystm/test/test_stmset.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/test/test_stmset.py
@@ -0,0 +1,85 @@
+
+
+class AppTestSet:
+ spaceconfig = dict(usemodules=['pypystm'])
+
+ def test_simple(self):
+ import pypystm
+ s = pypystm.stmset()
+ s.add(42.0)
+ assert 42.0 in s
+ assert 42.5 not in s
+ s.add(42.5)
+ assert 42.0 in s
+ assert 42.5 in s
+ s.add(42.5)
+ assert 42.0 in s
+ assert 42.5 in s
+ s.remove(42.0)
+ assert 42.0 not in s
+ assert 42.5 in s
+ raises(KeyError, s.remove, 42.0)
+ s.discard(42.0)
+ assert 42.0 not in s
+ assert 42.5 in s
+ s.discard(42.5)
+ assert 42.5 not in s
+
+ def test_hash_collision(self):
+ import pypystm
+ class Key(object):
+ def __hash__(self):
+ return 42
+ key1 = Key()
+ key2 = Key()
+ key3 = Key()
+ s = pypystm.stmset()
+ s.add(key1)
+ s.add(key2)
+ assert key1 in s
+ assert key2 in s
+ assert key3 not in s
+ s.remove(key1)
+ assert key1 not in s
+ assert key2 in s
+ assert key3 not in s
+ s.remove(key2)
+ assert key1 not in s
+ assert key2 not in s
+ assert key3 not in s
+
+ def test_must_be_hashable(self):
+ import pypystm
+ s = pypystm.stmset()
+ raises(TypeError, s.add, [])
+ raises(TypeError, s.remove, [])
+ raises(TypeError, s.discard, [])
+
+ def test_equal_elements(self):
+ import pypystm
+ s = pypystm.stmset()
+ s.add(42.0)
+ assert 42 in s
+ assert 42L in s
+ assert 42.001 not in s
+
+ def test_list_from_set(self):
+ import pypystm
+ s = pypystm.stmset()
+ assert len(s) == 0
+ assert tuple(s) == ()
+ s.add(42.5)
+ s.add(42.0)
+ assert sorted(s) == [42.0, 42.5]
+ assert len(s) == 2
+ s.remove(42.0)
+ assert list(s) == [42.5]
+ #
+ class Key(object):
+ def __hash__(self):
+ return hash(42.5)
+ key3 = Key()
+ s.add(key3)
+ assert len(s) == 2
+ items = list(s)
+ assert items == [42.5, key3] or items == [key3, 42.5]
diff --git a/pypy/module/_stm/test/test_time.py
b/pypy/module/pypystm/test/test_time.py
rename from pypy/module/_stm/test/test_time.py
rename to pypy/module/pypystm/test/test_time.py
--- a/pypy/module/_stm/test/test_time.py
+++ b/pypy/module/pypystm/test/test_time.py
@@ -1,13 +1,13 @@
class AppTestHashtable:
- spaceconfig = dict(usemodules=['_stm'])
+ spaceconfig = dict(usemodules=['pypystm'])
def test_simple(self):
- import _stm
- t1 = _stm.time()
- t2 = _stm.time()
+ import pypystm
+ t1 = pypystm.time()
+ t2 = pypystm.time()
assert t1 < t2 < t1 + 1
- t1 = _stm.clock()
- t2 = _stm.clock()
+ t1 = pypystm.clock()
+ t2 = pypystm.clock()
assert t1 < t2 < t1 + 1
diff --git a/pypy/module/pypystm/test_pypy_c/__init__.py
b/pypy/module/pypystm/test_pypy_c/__init__.py
new file mode 100644
diff --git a/pypy/module/pypystm/test_pypy_c/support.py
b/pypy/module/pypystm/test_pypy_c/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/test_pypy_c/support.py
@@ -0,0 +1,146 @@
+import py
+import sys, os, subprocess, types
+from rpython.tool.udir import udir
+
+
+class BaseTestSTM(object):
+
+ HEADER = """
+import thread, pypystm
+
+NUM_THREADS = 3
+
+_b_to_go = NUM_THREADS
+_b_done = False
+_b_lock = thread.allocate_lock()
+_b_locks = [thread.allocate_lock() for _i in range(NUM_THREADS)]
+for _bl in _b_locks:
+ _bl.acquire()
+
+class BarrierThreadsDone(Exception):
+ pass
+
+def barrier(tnum, done=False):
+ '''Waits until NUM_THREADS call this function, and then returns
+ in all these threads at once.'''
+ global _b_to_go, _b_done
+ _b_lock.acquire()
+ if done:
+ _b_done = True
+ _b_to_go -= 1
+ if _b_to_go > 0:
+ _b_lock.release()
+ _b_locks[tnum].acquire()
+ else:
+ _b_to_go = NUM_THREADS
+ for i in range(NUM_THREADS):
+ if i != tnum:
+ _b_locks[i].release()
+ _b_lock.release()
+ if _b_done:
+ raise BarrierThreadsDone
+
+def _run(tnum, lock, result, function, args):
+ start = pypystm.time()
+ try:
+ try:
+ while True:
+ function(*args)
+ if pypystm.time() - start >= 3.0:
+ break
+ except BarrierThreadsDone:
+ pass
+ result.append(1)
+ finally:
+ lock.release()
+ while len(result) != NUM_THREADS:
+ barrier(tnum, done=True)
+
+def run_in_threads(function, arg_thread_num=False, arg_class=None):
+ locks = []
+ result = []
+ for i in range(NUM_THREADS):
+ lock = thread.allocate_lock()
+ lock.acquire()
+ args = ()
+ if arg_thread_num:
+ args += (i,)
+ if arg_class:
+ args += (arg_class(),)
+ thread.start_new_thread(_run, (i, lock, result, function, args))
+ locks.append(lock)
+ for lock in locks:
+ lock._py3k_acquire(timeout=30)
+ if len(result) < len(locks):
+ raise Exception("not all threads completed successfully")
+"""
+
+ def setup_class(cls):
+ if '__pypy__' not in sys.builtin_module_names:
+ py.test.skip("must run this test with pypy")
+ try:
+ import pypystm
+ except ImportError:
+ py.test.skip("must give a pypy-c with stm enabled")
+ cls.tmpdir = udir.join('test-pypy-stm')
+ cls.tmpdir.ensure(dir=True)
+
+ def setup_method(self, meth):
+ self.filepath = self.tmpdir.join(meth.im_func.func_name + '.py')
+ self.logfile = self.filepath.new(ext='.log')
+
+ def _write_source(self, func_or_src, args=[]):
+ src = py.code.Source(func_or_src)
+ if isinstance(func_or_src, types.FunctionType):
+ funcname = func_or_src.func_name
+ else:
+ funcname = 'main'
+ arglist = ', '.join(map(repr, args))
+ with self.filepath.open("w") as f:
+ f.write(self.HEADER)
+ f.write(str(src) + '\n')
+ f.write("print %s(%s)\n" % (funcname, arglist))
+
+ def _execute(self, import_site=False):
+ cmdline = [sys.executable]
+ if not import_site:
+ cmdline.append('-S')
+ cmdline.append(str(self.filepath))
+ env = os.environ.copy()
+ env['PYPYSTM'] = str(self.logfile)
+ #
+ pipe = subprocess.Popen(cmdline,
+ env=env,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = pipe.communicate()
+ if pipe.returncode > 0:
+ raise IOError("subprocess error %d:\n%s" % (pipe.returncode,
+ stderr))
+ if pipe.returncode < 0:
+ raise IOError("subprocess was killed by signal %d" % (
+ pipe.returncode,))
+
+ def _parse_log(self):
+ from pypy.stm.print_stm_log import StmLog
+ return StmLog(str(self.logfile))
+
+ def _check_count_conflicts(self, func_or_src, args=[]):
+ self._write_source(func_or_src, args)
+ self._execute()
+ stmlog = self._parse_log()
+ count = stmlog.get_total_aborts_and_pauses()
+ print 'stmlog.get_total_aborts_and_pauses():', count
+ return count
+
+ def check_almost_no_conflict(self, *args):
+ count = self._check_count_conflicts(*args)
+ assert count < 500
+
+ def check_MANY_conflicts(self, *args):
+ count = self._check_count_conflicts(*args)
+ assert count > 20000
+
+ def check_SOME_conflicts(self, *args):
+ count = self._check_count_conflicts(*args)
+ assert count > 1000
diff --git a/pypy/module/pypystm/test_pypy_c/test_conflict.py
b/pypy/module/pypystm/test_pypy_c/test_conflict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/test_pypy_c/test_conflict.py
@@ -0,0 +1,46 @@
+from pypy.module.pypystm.test_pypy_c.support import BaseTestSTM
+
+
+class TestConflict(BaseTestSTM):
+
+ def test_obvious(self):
+ def f():
+ class X(object):
+ pass
+ x = X() # shared
+ x.a = 0
+ def g():
+ x.a += 1
+ run_in_threads(g)
+ #
+ self.check_MANY_conflicts(f)
+
+ def test_plain_dict_access(self):
+ def f():
+ d = {} # shared
+ def g(n):
+ d[n] = d.get(n, 0) + 1
+ run_in_threads(g, arg_thread_num=True)
+ #
+ self.check_MANY_conflicts(f)
+
+ def test_write_to_many_objects_in_order(self):
+ def f():
+ import weakref
+
+ class X(object):
+ pass
+
+ lst = [] # shared
+
+ def g(tnum):
+ if tnum == 0:
+ lst[:] = [X() for i in range(1000)]
+ barrier(tnum)
+ for x in lst:
+ x.a = 5
+ barrier(tnum)
+
+ run_in_threads(g, arg_thread_num=True)
+ #
+ self.check_SOME_conflicts(f)
diff --git a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py
b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py
@@ -0,0 +1,52 @@
+import py
+from pypy.module.pypystm.test_pypy_c.support import BaseTestSTM
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
