Following is my implementation, it's ok under my requirement. And for derived queries, I just start new queries based on different gql statements:
def GetRecords(date, begin, end, tags, bookmark, pagesize=100): """ http://google-appengine.googlegroups.com/web/efficient_paging_using_key_instead_of_a_dedicated_unique_property.txt *Original starting query* WHERE x > 0 AND x < 9 *Bookmarkable starting query* WHERE x > 0 AND x < 9 ORDER BY x ASC, __key__ ASC *Derived queries for starting from bookmark entity B* WHERE x = B.x AND __key__ > B ORDER BY __key__ ASC WHERE x > B.x AND x < 9 ORDER BY x ASC, __key__ ASC """ if 0 == len(tags): gqls = ["WHERE "+date+">=:1 AND "+date+"<=:2 ORDER BY "+date+" ASC, __key__ ASC", "WHERE "+date+"=:1 AND __key__>=:2 ORDER BY __key__ ASC", "WHERE "+date+">:1 AND "+date+"<=:2 ORDER BY "+date+" ASC, __key__ ASC" ] else: gqls = ["WHERE "+date+">=:1 AND "+date+"<=:2 AND tags=:3 ORDER BY "+date+" ASC, __key__ ASC", "WHERE "+date+"=:1 AND __key__>=:2 AND tags=:3 ORDER BY __key__ ASC", "WHERE "+date+">:1 AND "+date+"<=:2 AND tags=:3 ORDER BY "+date+" ASC, __key__ ASC" ] if bookmark is None: if 0 == len(tags): query = Record.gql(gqls[0], begin, end) else: query = Record.gql(gqls[0], begin, end, tags) records = query.fetch(pagesize+1) else: bookmarkv = {"inputtime": bookmark.inputtime, "date": bookmark.date} if 0 == len(tags): query1 = Record.gql(gqls[1], bookmarkv[date], bookmark.key()) query2 = Record.gql(gqls[2], bookmarkv[date], end) else: query1 = Record.gql(gqls[1], bookmarkv[date], bookmark.key(), tags) query2 = Record.gql(gqls[2], bookmarkv[date], end, tags) records = query1.fetch(pagesize+1) records += query2.fetch(pagesize+1-len(records)) if len(records)<pagesize+1: return records, None else: return records[:pagesize], records[-1] On 4月4日, 上午9时16分, Andrew Fong <[email protected]> wrote: > Correct me if I'm wrong, but it looks like you're trying to cram all > of this into one request-response cycle too -- this will run you afoul > of GAE's restrictions very quickly. > > You'll want to break down all those query iterations over multiple > request-response cycles. For normal paging, this is natural (viewing > 1000+ records on a page is very user-unfriendly), but if you're trying > to do a batch operation like updating every record in a table, you'll > want to break this down into small chunks (e.g. update the first 100 > rows, then the next 100, and so on) with each chunk corresponding to > one request. > > If you don't want to have to click a link several times, then just use > an AJAX call to iterate. The round-trip time for requests will slow > things down a bit, but big batch operations shouldn't be done that > frequently anyway (especially on App Engine). > > -- Andrew > > On Apr 3, 5:56 pm, Jeff S <[email protected]> wrote: > > > > > When Ryan mentioned "derived queries" he meant, use new queries that > > will pick up where the last query left off (also called paging as you > > noted). Joe Gregorio wrote an article on it which might also help: > > >http://code.google.com/appengine/articles/paging.html > > > This article describes a few different approaches you can use, one of > > which is using __key__ comparisons. > > > Happy coding, > > > Jeff > > > On Apr 2, 9:09 am, 秦锋 <[email protected]> wrote: > > > > Hi all: > > > I'm tried to read all records and find tags of them, since my records > > > are larger than 1000(>4000 actually), and I know fetch has 1000 limit, > > > thus I tried iterator, but got failure either. > > > Following is my code: > > > > class UpdateTag(webapp.RequestHandler): > > > def get(self): > > > if 0 == len(self.request.get('begin')): > > > begin = datetime.datetime.strptime("197001010000", "%Y%m%d%H%M > > > %S") > > > else: > > > begin = datetime.datetime.strptime(self.request.get('begin'), "%Y > > > %m%d%H%M%S") > > > if 0 == len(self.request.get('end')): > > > end = datetime.datetime.now() > > > else: > > > end = datetime.datetime.strptime(self.request.get('end'), "%Y%m%d > > > %H%M%S") > > > query = statsdb.Record.all().filter("inputtime >=", begin).filter > > > ("inputtime <=", end) > > > self.response.headers['Content-Type'] = 'text/plain' > > > > tagsName = [] > > > i = 0 > > > for record in query: > > > tagsName = tagsName + list(set(record.tags)-set(tagsName)) > > > i+=1 > > > if 0 == len(tagsName): > > > self.response.out.write("No records counted.\n") > > > else: > > > self.response.out.write(str(i) + " records counted.\n") > > > > tags = {} > > > for tagName in tagsName: > > > tag = statsdb.Tags(key_name = tagName, name = tagName) > > > bookmark = None > > > pagesize = 100 > > > while True: > > > records, bookmark = statsdb.GetRecords([tagName], bookmark, > > > pagesize) > > > if bookmark is None: > > > tag.refCount += len(records) > > > break > > > else: > > > tag.refCount += pagesize > > > tags[tagName] = tag > > > batch = 100 > > > if len(self.request.get('batch')) > 0: > > > batch = int(self.request.get('batch')) > > > statsdb.DBPut(tags.values(), batch) > > > self.response.out.write(str(len(tags)) + " tags counted.\n") > > > > And this is error: > > > <pre>Traceback (most recent call last): > > > File "/base/python_lib/versions/1/google/appengine/ext/webapp/ > > > __init__.py", line 501, in __call__ > > > handler.get(*groups) > > > File "/base/data/home/apps/cndata4u/1.332506907084827982/ > > > dbmaint.py", line 26, in get > > > for record in query: > > > File "/base/python_lib/versions/1/google/appengine/ext/db/ > > > __init__.py", line 1468, in next > > > return self.__model_class.from_entity(self.__iterator.next()) > > > File "/base/python_lib/versions/1/google/appengine/api/ > > > datastore.py", line 1549, in next > > > self.__buffer = self._Next(self._BUFFER_SIZE) > > > File "/base/python_lib/versions/1/google/appengine/api/ > > > datastore.py", line 1538, in _Next > > > raise _ToDatastoreError(err) > > > File "/base/python_lib/versions/1/google/appengine/api/ > > > datastore.py", line 1965, in _ToDatastoreError > > > raise errors[err.application_error](err.error_detail) > > > Timeout: datastore timeout: operation took too long. > > > </pre> > > > > I know there are some way to page records, but only apply I have no > > > extra inequality in property since __key__ will occur that. And in > > > this case I have already an inequality, how to proceed? > > > I don't understand a little bit about method > > > here:http://groups.google.com/group/google-appengine/browse_thread/thread/... > > > and what's meaning about "derived queries"? --~--~---------~--~----~------------~-------~--~----~ You received this message because you are subscribed to the Google Groups "Google App Engine" group. To post to this group, send email to [email protected] To unsubscribe from this group, send email to [email protected] For more options, visit this group at http://groups.google.com/group/google-appengine?hl=en -~----------~----~----~----~------~----~------~--~---
