Charles-François Natali <neolo...@free.fr> added the comment:
Here's a patch based on the second approach.
----------
Added file: http://bugs.python.org/file22490/heap_gc_deadlock_lockless.diff
_______________________________________
Python tracker <rep...@bugs.python.org>
<http://bugs.python.org/issue12352>
_______________________________________
diff -r fca745bc70be Lib/multiprocessing/heap.py
--- a/Lib/multiprocessing/heap.py Sat Jun 25 16:31:06 2011 +0200
+++ b/Lib/multiprocessing/heap.py Sun Jun 26 19:57:56 2011 +0200
@@ -101,6 +101,8 @@
self._stop_to_block = {}
self._allocated_blocks = set()
self._arenas = []
+ # list of pending blocks to free - see free() comment below
+ self._pending_free_blocks = []
@staticmethod
def _roundup(n, alignment):
@@ -175,15 +177,39 @@
return start, stop
+ def _free_pending_blocks(self):
+ # Free all the blocks in the pending list - called with the lock held.
+ while True:
+ try:
+ block = self._pending_free_blocks.pop()
+ except IndexError:
+ break
+ self._free(block)
+ self._allocated_blocks.remove(block)
+
def free(self, block):
# free a block returned by malloc()
+ # Since free() can be called asynchronously by the GC, it could happen
+ # that it's called while self._lock is held: in that case,
+ # self._lock.acquire() would deadlock (issue #12352). To avoid that, a
+ # trylock is used instead, and if the lock can't be acquired
+ # immediately, the block is added to a list of blocks to be freed
+ # synchronously sometimes later from malloc() or free(), by calling
+ # _free_pending_blocks() (appending and retrieving from a list is not
+ # strictly thread-safe but under cPython it's atomic thanks to the
GIL).
assert os.getpid() == self._lastpid
- self._lock.acquire()
- try:
- self._allocated_blocks.remove(block)
- self._free(block)
- finally:
- self._lock.release()
+ if not self._lock.acquire(0):
+ # can't acquire the lock, add it to the list of pending blocks to
+ # free
+ self._pending_free_blocks.append(block)
+ else:
+ # we hold the lock
+ try:
+ self._free_pending_blocks()
+ self._allocated_blocks.remove(block)
+ self._free(block)
+ finally:
+ self._lock.release()
def malloc(self, size):
# return a block of right size (possibly rounded up)
@@ -191,6 +217,7 @@
if os.getpid() != self._lastpid:
self.__init__() # reinitialize after fork
self._lock.acquire()
+ self._free_pending_blocks()
try:
size = self._roundup(max(size,1), self._alignment)
(arena, start, stop) = self._malloc(size)
diff -r fca745bc70be Lib/test/test_multiprocessing.py
--- a/Lib/test/test_multiprocessing.py Sat Jun 25 16:31:06 2011 +0200
+++ b/Lib/test/test_multiprocessing.py Sun Jun 26 19:57:56 2011 +0200
@@ -1737,7 +1737,31 @@
(narena, nstart, nstop) = all[i+1][:3]
self.assertTrue((arena != narena and nstart == 0) or
(stop == nstart))
-
+
+ def test_free_from_gc(self):
+ # Check that freeing of blocks by the garbage collector doesn't
deadlock
+ # (issue #12352).
+ # Make sure the GC is enabled, and set lower collection thresholds to
+ # make collections more frequent (and increase the probability of
+ # deadlock).
+ if gc.isenabled():
+ thresholds = gc.get_threshold()
+ self.addCleanup(gc.set_threshold, *thresholds)
+ else:
+ gc.enable()
+ self.addCleanup(gc.disable)
+ gc.set_threshold(10)
+
+ # perform numerous block allocations, with cyclic references to make
+ # sure objects are collected asynchronously by the gc
+ for i in range(5000):
+ a = multiprocessing.heap.BufferWrapper(1)
+ b = multiprocessing.heap.BufferWrapper(1)
+ # circular references
+ a.buddy = b
+ b.buddy = a
+
+
#
#
#
_______________________________________________
Python-bugs-list mailing list
Unsubscribe:
http://mail.python.org/mailman/options/python-bugs-list/archive%40mail-archive.com
