Hi,
I've gone through the Queue class and done a number
of optimizations that resulted in a very significant speed
increase. The modified verion and a test script are
attached. I've only done a very simple test so far.
prior to optimizations using the existing Queue class
======================================
tavis@lucy: ~ > time python test.py
real 0m6.367s
user 0m6.260s
sys 0m0.020s
after to optimizations
======================================
tavis@lucy: ~ > time python test.py
real 0m4.361s
user 0m4.320s
sys 0m0.020s
Difference
======================================
It shaves off 2.006 seconds (31%) from the original time
This could be worthwhile considering how central
queues are in WebKit.
Tavis
p.s. sourceforge's mail server is taking hours
to deliver things today. anyone know why?
That's why I cc'd all of you...
"""A multi-producer, multi-consumer queue."""
class Empty(Exception):
"Exception raised by Queue.get(block=0)/get_nowait()."
pass
class Full(Exception):
"Exception raised by Queue.put(block=0)/put_nowait()."
pass
class Queue:
def __init__(self, maxsize=0):
"""Initialize a queue object with a given maximum size.
If maxsize is <= 0, the queue size is infinite.
"""
import thread
self._init(maxsize)
self.mutex = thread.allocate_lock()
self.esema = thread.allocate_lock()
self.esema.acquire()
self.fsema = thread.allocate_lock()
##shortcut name-bindings
self.mutexAcquire = self.mutex.acquire
self.mutexRelease = self.mutex.release
self.fsemaAcquire = self.fsema.acquire
self.fsemaRelease = self.fsema.release
self.esemaAcquire = self.esema.acquire
self.esemaRelease = self.esema.release
self._put = self.queue.append
def qsize(self):
"""Return the approximate size of the queue (not reliable!)."""
self.mutexAcquire()
n = len(self.queue)
self.mutexRelease()
return n
def empty(self):
"""Return 1 if the queue is empty, 0 otherwise (not reliable!)."""
self.mutexAcquire()
n = not self.queue
self.mutexRelease()
return n
def full(self):
"""Return 1 if the queue is full, 0 otherwise (not reliable!)."""
self.mutexAcquire()
n = ( self.maxsize > 0 and len(self.queue) == self.maxsize )
self.mutexRelease()
return n
def put(self, item, block=1):
"""Put an item into the queue.
If optional arg 'block' is 1 (the default), block if
necessary until a free slot is available. Otherwise (block
is 0), put an item on the queue if a free slot is immediately
available, else raise the Full exception.
"""
theQueue = self.queue
if block:
self.fsemaAcquire()
elif not self.fsemaAcquire(0):
raise Full
self.mutexAcquire()
was_empty = not theQueue # ~= self._empty()
self._put(item)
if was_empty:
self.esemaRelease()
if not (self.maxsize > 0 and len(theQueue) == self.maxsize): # ~= not
self._full():
self.fsemaRelease()
self.mutexRelease()
def put_nowait(self, item):
"""Put an item into the queue without blocking.
Only enqueue the item if a free slot is immediately available.
Otherwise raise the Full exception.
"""
return self.put(item, 0)
def get(self, block=1):
"""Remove and return an item from the queue.
If optional arg 'block' is 1 (the default), block if
necessary until an item is available. Otherwise (block is 0),
return an item if one is immediately available, else raise the
Empty exception.
"""
theQueue = self.queue
if block:
self.esemaAcquire()
elif not self.esemaAcquire(0):
raise Empty
self.mutexAcquire()
was_full = (self.maxsize > 0 and len(theQueue) == self.maxsize) # ~=
self._full()
item = theQueue[0]
del theQueue[0]
if was_full:
self.fsemaRelease()
if theQueue: # ~= not self._empty():
self.esemaRelease()
self.mutexRelease()
return item
def get_nowait(self):
"""Remove and return an item from the queue without blocking.
Only get an item if one is immediately available. Otherwise
raise the Empty exception.
"""
return self.get(0)
# Initialize the queue representation
def _init(self, maxsize):
self.maxsize = maxsize
self.queue = []
#from Queue import Queue, Full, Empty
from Queue2 import Queue, Full, Empty
MAXSIZE = 1000
q = Queue(MAXSIZE)
def putThenGet():
for i in range(MAXSIZE):
q.put(i)
for i in range(MAXSIZE):
val = q.get()
if __name__ == '__main__':
for i in range(100):
putThenGet()
