Stefan van der Walt wrote:
On Wed, Nov 15, 2006 at 02:33:52PM -0600, Robert Kern wrote:
Mathew Yeates wrote:
Hi
I'm running a 64 bit Python 2.5 on an x86 with Solaris. I have a
function I call over 2^32 times and eventually I run out of memory.
The function is
def make_B(deltadates):
numcols=deltadates.shape[0]
B=numpy.zeros((numcols,numcols))
for ind in range(0,numcols): #comment out this loop and all is good
B[ind,0:numcols] = deltadates[0:numcols]
return B
If I comment out the loop lines, my memory is okay. I'm guessing that a
reference is being added to "deltadates" and that the reference count is
going above 2^32 and reseting. Anybody have any ideas about how I can
cure this? Is Numpy increasing the reference count here?
Can you give us a small but complete and self-contained script that demonstrates
the problem?
I think this might be related to ticket #378:
http://projects.scipy.org/scipy/numpy/ticket/378
Cheers
Stéfan
okay. attached is the smallest program I could make. Before running you
will need to create a file named biggie with 669009000 non zero floats.
Mathew
#program leaks memory on a 64 bit Python2.5 x86 Solaris
import numpy
import datetime
def make_B(deltadates):
numcols=deltadates.shape[0]
B=numpy.zeros((numcols,numcols))
for ind in range(0,numcols):
B[ind,0:numcols] = deltadates[0:numcols]
return B
def delta2day1(delta):
return delta.days/365.0
deltas2days=numpy.frompyfunc(delta2day1,1,1)
def get_connected_dates(numdates,data,all_dates,start_dates,
end_dates,refdate):
datecovered=numpy.array([False]*numdates)
for ind in range(0,data.shape[0]):
if data[ind] != 0:
starti=all_dates.index(start_dates[ind])
endi=all_dates.index(end_dates[ind])
datecovered[starti:endi+1] = True
refi=all_dates.index(refdate)
if not datecovered[refi]:
return False
after_list = refi + numpy.where(numpy.logical_and.accumulate(
datecovered[refi:] != False))[0]
reversei= numdates - refi
before_list = refi -1 - numpy.where(numpy.logical_and.accumulate(
datecovered[::-1][reversei:] != False))[0]
before_list.sort()
contig=numpy.concatenate((before_list,after_list),axis=1)
usedates=numpy.array(all_dates)[contig]
return usedates,contig
datasize=5529000
cols=45
iterlist=range(0,datasize)
curpercent=-2
filecount=121
refdate=datetime.date(1996, 4, 6)
numdates=46
all_dates=[datetime.date(1992, 8, 26), datetime.date(1993, 4, 28),
datetime.date(1993, 10, 20), datetime.date(1995, 4, 21), datetime.date(1995, 5,
26), datetime.date(1995, 6, 30), datetime.date(1995, 8, 4), datetime.date(1995,
9, 8), datetime.date(1995, 10, 13), datetime.date(1995, 11, 18),
datetime.date(1995, 12, 22), datetime.date(1995, 12, 23), datetime.date(1996,
1, 26), datetime.date(1996, 1, 27), datetime.date(1996, 4, 5),
datetime.date(1996, 4, 6), datetime.date(1996, 5, 11), datetime.date(1996, 6,
15), datetime.date(1996, 8, 24), datetime.date(1996, 9, 28),
datetime.date(1997, 1, 11), datetime.date(1997, 4, 26), datetime.date(1997, 9,
13), datetime.date(1997, 10, 18), datetime.date(1997, 11, 22),
datetime.date(1998, 1, 31), datetime.date(1998, 3, 7), datetime.date(1998, 4,
11), datetime.date(1998, 5, 16), datetime.date(1998, 6, 20),
datetime.date(1998, 7, 25), datetime.date(1998, 8, 29), datetime.date(1998, 10,
3), datetime.date(1998, 11, 7), datetime.date(1998, 12, 12),
datetime.date(1999, 1, 16), datetime.date(1999, 5, 1), datetime.date(1999, 6,
5), datetime.date(1999, 10, 23), datetime.date(1999, 11, 27),
datetime.date(2000, 2, 5), datetime.date(2000, 3, 11), datetime.date(2000, 4,
15), datetime.date(2000, 5, 20), datetime.date(2000, 6, 24),
datetime.date(2000, 11, 11)]
start_dates=[datetime.date(1992, 8, 26), datetime.date(1992, 8, 26),
datetime.date(1992, 8, 26), datetime.date(1993, 4, 28), datetime.date(1993, 4,
28), datetime.date(1993, 10, 20), datetime.date(1995, 4, 21),
datetime.date(1995, 4, 21), datetime.date(1995, 5, 26), datetime.date(1995, 5,
26), datetime.date(1995, 8, 4), datetime.date(1995, 8, 4), datetime.date(1995,
10, 13), datetime.date(1995, 12, 22), datetime.date(1995, 12, 23),
datetime.date(1995, 12, 23), datetime.date(1995, 12, 23), datetime.date(1996,
1, 26), datetime.date(1996, 4, 6), datetime.date(1996, 5, 11),
datetime.date(1996, 8, 24), datetime.date(1996, 9, 28), datetime.date(1997, 1,
11), datetime.date(1997, 1, 11), datetime.date(1997, 4, 26),
datetime.date(1997, 9, 13), datetime.date(1997, 9, 13), datetime.date(1997, 11,
22), datetime.date(1997, 11, 22), datetime.date(1997, 11, 22),
datetime.date(1998, 5, 16), datetime.date(1998, 5, 16), datetime.date(1998, 5,
16), datetime.date(1998, 6, 20), datetime.date(1998, 6, 20),
datetime.date(1998, 7, 25), datetime.date(1998, 10, 3), datetime.date(1998, 12,
12), datetime.date(1999, 1, 16), datetime.date(1999, 1, 16),
datetime.date(1999, 6, 5), datetime.date(1999, 10, 23), datetime.date(1995, 4,
21), datetime.date(1996, 9, 28), datetime.date(1998, 1, 31),
datetime.date(1998, 4, 11), datetime.date(1998, 7, 25), datetime.date(1996, 6,
15), datetime.date(1996, 9, 28), datetime.date(1997, 4, 26),
datetime.date(1998, 4, 11), datetime.date(1998, 7, 25), datetime.date(1997, 1,
11), datetime.date(1997, 9, 13), datetime.date(1998, 1, 31),
datetime.date(1998, 4, 11), datetime.date(1998, 10, 3), datetime.date(1996, 8,
24), datetime.date(1998, 3, 7), datetime.date(1997, 9, 13), datetime.date(1997,
10, 18), datetime.date(1998, 1, 31), datetime.date(1992, 8, 26),
datetime.date(1993, 4, 28), datetime.date(1993, 10, 20), datetime.date(1995, 6,
30), datetime.date(1995, 6, 30), datetime.date(1995, 8, 4), datetime.date(1995,
8, 4), datetime.date(1995, 9, 8), datetime.date(1995, 9, 8),
datetime.date(1995, 10, 13), datetime.date(1995, 12, 22), datetime.date(1995,
12, 23), datetime.date(1996, 1, 26), datetime.date(1996, 1, 27),
datetime.date(1996, 4, 6), datetime.date(1996, 5, 11), datetime.date(1996, 6,
15), datetime.date(1996, 6, 15), datetime.date(1996, 6, 15),
datetime.date(1996, 8, 24), datetime.date(1996, 9, 28), datetime.date(1997, 1,
11), datetime.date(1997, 1, 11), datetime.date(1997, 4, 26),
datetime.date(1997, 10, 18), datetime.date(1998, 4, 11), datetime.date(1998, 4,
11), datetime.date(1998, 4, 11), datetime.date(1998, 7, 25),
datetime.date(1998, 7, 25), datetime.date(1998, 7, 25), datetime.date(1999, 1,
16), datetime.date(1999, 5, 1), datetime.date(1999, 5, 1), datetime.date(1995,
4, 21), datetime.date(1995, 10, 13), datetime.date(1995, 12, 22),
datetime.date(1996, 1, 26), datetime.date(1996, 1, 26), datetime.date(1996, 1,
26), datetime.date(1996, 4, 5), datetime.date(1996, 4, 5), datetime.date(1996,
4, 6), datetime.date(1996, 4, 6), datetime.date(1996, 5, 11),
datetime.date(1996, 5, 11), datetime.date(1996, 6, 15), datetime.date(1996, 9,
28), datetime.date(1998, 1, 31), datetime.date(1998, 1, 31),
datetime.date(1999, 10, 23), datetime.date(1999, 10, 23), datetime.date(2000,
4, 15), datetime.date(2000, 2, 5), datetime.date(2000, 2, 5),
datetime.date(1998, 3, 7), datetime.date(1995, 11, 18), datetime.date(1998, 8,
29), datetime.date(1995, 12, 22)]
end_dates=[datetime.date(1995, 8, 4), datetime.date(1996, 1, 27),
datetime.date(1996, 4, 5), datetime.date(1995, 10, 13), datetime.date(1995, 12,
22), datetime.date(1995, 10, 13), datetime.date(1995, 5, 26),
datetime.date(1996, 4, 5), datetime.date(1996, 4, 5), datetime.date(1996, 4,
6), datetime.date(1996, 1, 26), datetime.date(1996, 1, 27), datetime.date(1998,
11, 7), datetime.date(1998, 10, 3), datetime.date(1996, 1, 27),
datetime.date(1996, 5, 11), datetime.date(1997, 11, 22), datetime.date(1996, 1,
27), datetime.date(2000, 2, 5), datetime.date(1997, 11, 22),
datetime.date(1998, 3, 7), datetime.date(2000, 4, 15), datetime.date(2000, 11,
11), datetime.date(1997, 9, 13), datetime.date(1998, 3, 7), datetime.date(1997,
10, 18), datetime.date(1998, 8, 29), datetime.date(2000, 5, 20),
datetime.date(1998, 6, 20), datetime.date(1998, 8, 29), datetime.date(2000, 5,
20), datetime.date(1998, 6, 20), datetime.date(1999, 6, 5), datetime.date(2000,
5, 20), datetime.date(1999, 6, 5), datetime.date(1999, 5, 1),
datetime.date(1999, 6, 5), datetime.date(2000, 6, 24), datetime.date(1999, 5,
1), datetime.date(1999, 10, 23), datetime.date(2000, 5, 20),
datetime.date(1999, 11, 27), datetime.date(1996, 4, 6), datetime.date(2000, 2,
5), datetime.date(2000, 2, 5), datetime.date(2000, 2, 5), datetime.date(2000,
2, 5), datetime.date(2000, 3, 11), datetime.date(2000, 3, 11),
datetime.date(2000, 3, 11), datetime.date(2000, 3, 11), datetime.date(2000, 3,
11), datetime.date(2000, 4, 15), datetime.date(2000, 4, 15),
datetime.date(2000, 4, 15), datetime.date(2000, 4, 15), datetime.date(2000, 5,
20), datetime.date(2000, 6, 24), datetime.date(2000, 6, 24),
datetime.date(2000, 11, 11), datetime.date(2000, 11, 11), datetime.date(2000,
11, 11), datetime.date(1996, 1, 26), datetime.date(1993, 10, 20),
datetime.date(1995, 12, 22), datetime.date(1995, 9, 8), datetime.date(1998, 12,
12), datetime.date(1995, 12, 23), datetime.date(1996, 5, 11),
datetime.date(1995, 11, 18), datetime.date(1998, 12, 12), datetime.date(1998,
10, 3), datetime.date(1999, 6, 5), datetime.date(1998, 5, 16),
datetime.date(1996, 4, 5), datetime.date(1996, 5, 11), datetime.date(1996, 9,
28), datetime.date(1998, 5, 16), datetime.date(1997, 4, 26),
datetime.date(1998, 3, 7), datetime.date(1998, 4, 11), datetime.date(1998, 12,
12), datetime.date(1998, 1, 31), datetime.date(1997, 10, 18),
datetime.date(1998, 8, 29), datetime.date(1998, 4, 11), datetime.date(1998, 8,
29), datetime.date(1999, 1, 16), datetime.date(1999, 5, 1), datetime.date(1999,
11, 27), datetime.date(1999, 1, 16), datetime.date(1999, 10, 23),
datetime.date(1999, 11, 27), datetime.date(1999, 11, 27), datetime.date(1999,
10, 23), datetime.date(1999, 11, 27), datetime.date(1996, 9, 28),
datetime.date(1995, 12, 22), datetime.date(1996, 5, 11), datetime.date(1997,
10, 18), datetime.date(1997, 11, 22), datetime.date(1998, 8, 29),
datetime.date(1996, 4, 6), datetime.date(1997, 1, 11), datetime.date(1997, 1,
11), datetime.date(1998, 1, 31), datetime.date(1998, 10, 3),
datetime.date(1999, 6, 5), datetime.date(1996, 9, 28), datetime.date(1997, 4,
26), datetime.date(1998, 4, 11), datetime.date(1998, 7, 25),
datetime.date(2000, 11, 11), datetime.date(2000, 4, 15), datetime.date(2000,
11, 11), datetime.date(2000, 4, 15), datetime.date(2000, 3, 11),
datetime.date(2000, 3, 11), datetime.date(1998, 12, 12), datetime.date(2000,
11, 11), datetime.date(1998, 6, 20)]
#biggie is a file with 669009000 floats. The values are unimportant as long as
they are not 0
diff_phases=numpy.memmap("biggie",mode="r+",
shape=(datasize,filecount),dtype=numpy.float32)
for row in iterlist:
data = diff_phases[row,:]
data=numpy.nan_to_num(data) #set Nans to zero
if len(data[numpy.where(data != 0)]) == 0: continue #no good data
res=get_connected_dates(numdates,data,all_dates,start_dates,
end_dates,refdate)
if not res: continue
usedates,contig=res
if usedates.shape[0] == 0:continue
deltadates=usedates[1:] - usedates[:-1]
deltadates=deltas2days(deltadates)
B = make_B(deltadates)
percent=(100.0 * row)/datasize
if percent >= curpercent + 1:
print "%f percent done" % percent
curpercent=percent
-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys - and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
_______________________________________________
Numpy-discussion mailing list
Numpy-discussion@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/numpy-discussion
