I think my most recent message might have been unclear. The simple script
included in my earlier note appears to demonstrate a memory leak, even with
regular calls to store.commit(). (NB: not "db.commit()").
Here is a more complicated script that attempts to purge from memory any
leftover state. (It also prints out its current memory usage as it runs.) It,
too, continually leaks memory until it uses up all system memory and is killed.
Regards,
Zooko
-------
import gc, os, random, sys, time
import metakit
print metakit, metakit.__version__
def get_mem_usage():
"""
This only works on Linux, and only if the /proc/$PID/stat output is the
same as that in linux kernel 2.4. Also `os.getpid()' must work.
@returns the virtual memory used by this process
"""
# sample output from cat /proc/$PID/stat:
# 19606 (python2.2) S 19592 19592 7451 34834 19592 0 2155 0 970 0 57 12 0 0
9 0 0 0 183707687 36585472 5206 4294967295 134512640 135126528 3221221280
3221219220 1074927566 0 0 4096 137912326 3222525185 0 0 17 0
a = os.popen("cat /proc/%s/stat" % os.getpid()).read().split()
assert a[0] == "%s" % os.getpid()
return int(a[22])
store = None
db = None
USE_HASH_VIEW=False
def init_mk():
global store, db
store = metakit.storage("whee", 1)
if USE_HASH_VIEW:
_FORMAT = "test_dbB[uniqid:B,thing:I]"
hashvw = store.getas("test_hash[_H:I,_R:I]")
db = store.getas(_FORMAT).hash(hashvw, 1)
else:
_FORMAT = "test_db[_B[uniqid:B,thing:I]]"
# else, use blocked, ordered view
db = store.getas(_FORMAT).blocked().ordered()
def insert_mk(N):
for i in range(N):
uniqid = os.urandom(20)
thing = random.randrange(0, 2**20)
dbhit = db.find(uniqid=uniqid)
assert dbhit == -1
db.insert(dbhit, uniqid=uniqid, thing=thing)
store.commit()
gc.collect()
def teardown_mk():
global db, store
db = None
store.commit()
store = None
gc.collect()
os.remove("whee")
while True:
init_mk()
print "before -- % 10.f bytes" % get_mem_usage(),
insert_mk(2**18)
print "after -- % 10.f bytes" % get_mem_usage(),
teardown_mk()
print "clean -- % 10.f bytes" % get_mem_usage()
#!/usr/bin/env python
import gc, os, random, sys, time
import metakit
print metakit, metakit.__version__
def get_mem_usage():
"""
This only works on Linux, and only if the /proc/$PID/stat output is the
same as that in linux kernel 2.4. Also `os.getpid()' must work.
@returns the virtual memory used by this process
"""
# sample output from cat /proc/$PID/stat:
# 19606 (python2.2) S 19592 19592 7451 34834 19592 0 2155 0 970 0 57 12 0 0
9 0 0 0 183707687 36585472 5206 4294967295 134512640 135126528 3221221280
3221219220 1074927566 0 0 4096 137912326 3222525185 0 0 17 0
a = os.popen("cat /proc/%s/stat" % os.getpid()).read().split()
assert a[0] == "%s" % os.getpid()
return int(a[22])
store = None
db = None
USE_HASH_VIEW=False
def init_mk():
global store, db
store = metakit.storage("whee", 1)
if USE_HASH_VIEW:
_FORMAT = "test_dbB[uniqid:B,thing:I]"
hashvw = store.getas("test_hash[_H:I,_R:I]")
db = store.getas(_FORMAT).hash(hashvw, 1)
else:
_FORMAT = "test_db[_B[uniqid:B,thing:I]]"
# else, use blocked, ordered view
db = store.getas(_FORMAT).blocked().ordered()
def insert_mk(N):
for i in range(N):
uniqid = os.urandom(20)
thing = random.randrange(0, 2**20)
dbhit = db.find(uniqid=uniqid)
assert dbhit == -1
db.insert(dbhit, uniqid=uniqid, thing=thing)
store.commit()
gc.collect()
def teardown_mk():
global db, store
db = None
store.commit()
store = None
gc.collect()
os.remove("whee")
while True:
init_mk()
print "before -- % 10.f bytes" % get_mem_usage(),
insert_mk(2**18)
print "after -- % 10.f bytes" % get_mem_usage(),
teardown_mk()
print "clean -- % 10.f bytes" % get_mem_usage()
_____________________________________________
Metakit mailing list - Metakit@equi4.com
http://www.equi4.com/mailman/listinfo/metakit
