Hi!
I'm trying to store a lot of polygons (along with the points contained
within its bounding box). I have a task handler
(AnalyzePolygonHandler) that does it. I frequently run into this
error, "Exceeded soft process size limit with 250.059 MB after
servicing 33 requests total."
I create a huge list, but it only gets up to a few hundred thousand
items (5-20 character strings) max. Using sys.getsizeof(), the list,
at about a million items which I rarely reach, is only 8MB in size.
I split the items in the list and stored them in a list property on
multiple entities, each containing 1000 items.
The code is below. Why am I running into this error?
class AnalyzePolygonHandler(webapp.RequestHandler):
def post(self):
polygon = simplejson.loads(self.request.get('polygon'))
key = polygon["key"].strip()
points = polygon["points"]
#convert points to tuples
coords = []
for point in points:
coords.append((float(point["x"]),float(point["y"])))
#commit!!!
points = analyzepolygon(coords)
models = []
#put only 1000 items inside 1 model
incr = 1000
total = len(points)
current = 0
coords_json = simplejson.dumps(coords)
while current < total:
model = P()
model.p = points[:incr]
del points[:incr]
model.n = key
model.c = coords_json
models.append(model)
current += incr
try:
db.put(models)
logging.error("Success in putting " + str(len(models)) + "
entities.")
except:
logging.exception("DB PUT ERROR!!!")
def analyzepolygon(polygon):
"""
Receives a polygon (list of coordinate tuples). eg. [(x1,y1),
(x2,y2),(x3,y3),(x1,y1)]
Returns a list of geocodes within the bounding box of the polygon.
"""
#get bounding box
min_x = polygon[0][0]
max_x = polygon[0][0]
min_y = polygon[0][1]
max_y = polygon[0][1]
for coord in polygon:
if coord[0] < min_x:
min_x = coord[0]
if coord[0] > max_x:
max_x = coord[0]
if coord[1] < min_y:
min_y = coord[1]
if coord[1] > max_y:
max_y = coord[1]
#test with increasing resolutions
r = 0 #resolution. at start at 0, and then gradually increase
until the polygon's area is at least 1000 points
total = 0
power = 10**r
while total < 10000:
power = 10**r
temp_max_x = int(max_x * power)
temp_min_x = int(min_x * power)
temp_max_y = int(max_y * power)
temp_min_y = int(min_y * power)
total = (temp_max_x - temp_min_x) * (temp_max_y - temp_min_y)
if total < 0:
total = total * -1
r += 1
#generate points (integer type)
points = []
incr = 1
places = r - 1
#convert to integer
max_x = int(max_x * power)
min_x = int(min_x * power)
max_y = int(max_y * power)
min_y = int(min_y * power)
#traverse!!!
x = min_x
y = min_y
while x <= max_x:
while y <= max_y:
points.append(geoencode(x,y,decimal_places))
y += incr
y = min_y
x += incr
#return the geoencoded list
return points
def geoencode(x,y,places):
if x < 0 and y < 0:
sign = "b"
x *= -1
y *= -1
elif x < 0:
sign = "c"
x *= -1
elif y < 0:
sign = "d"
y *= -1
else:
sign = "a"
x = str(x)
y = str(y)
len_x = len(x)
len_y = len(y)
if len_x <= places:
difference = (places + 1) - len_x
for b in range(difference):
x = "0" + x
if len_y <= places:
difference = (places + 1) - len_y
for b in range(difference):
y = "0" + y
if places == 0:
base_x = x
base_y = y
else:
base_x = x[:(places*-1)]
base_y = y[:(places*-1)]
current_x = len(base_x)
current_y = len(base_y)
encoded_str = sign + base_x + "," + base_y
if places > 0:
encoded_str = encoded_str + ","
i = 0
while i < places:
encoded_str = encoded_str + x[current_x:current_x+1] +
y[current_y:current_y+1]
current_x += 1
current_y += 1
i += 1
return encoded_str
--
You received this message because you are subscribed to the Google Groups
"Google App Engine" group.
To post to this group, send email to [email protected].
To unsubscribe from this group, send email to
[email protected].
For more options, visit this group at
http://groups.google.com/group/google-appengine?hl=en.
