Revision: 7360
Author: alexsh
Date: 2009-10-03 08:54:22 +0000 (Sat, 03 Oct 2009)
Log Message:
-----------
use large data post array to query redirects
Modified Paths:
--------------
trunk/pywikipedia/query.py
trunk/pywikipedia/redirect.py
Modified: trunk/pywikipedia/query.py
===================================================================
--- trunk/pywikipedia/query.py 2009-10-03 04:52:39 UTC (rev 7359)
+++ trunk/pywikipedia/query.py 2009-10-03 08:54:22 UTC (rev 7360)
@@ -59,18 +59,19 @@
# Titles param might be long, case convert it to post request
data = None
titlecount = 0
- if 'titles' in params:
- titlecount = params['titles'].count('|')
- if encodeTitle:
- data = {'titles' : params['titles']}
- del params['titles']
+ for pLongKey in ['titles', 'pageids', 'ucusers']: #
+ if pLongKey in params:
+ titlecount = params[pLongKey].count('|')
+ if encodeTitle:
+ data = {pLongKey : params[pLongKey]}
+ del params[pLongKey]
postAC = [
'edit', 'login', 'purge', 'rollback', 'delete', 'undelete', 'protect',
'block', 'unblock', 'move', 'emailuser','import', 'userrights',
]
if useAPI:
- if params['action'] in postAC or data:
+ if params['action'] in postAC:
path = site.api_address()
else:
path = site.api_address() + urllib.urlencode(params.items())
@@ -80,7 +81,7 @@
if wikipedia.verbose:
if titlecount > 0:
- wikipedia.output(u"Requesting %d titles from %s:%s" % (titlecount,
site.lang, path))
+ wikipedia.output(u"Requesting %d %s from %s:%s" % (titlecount,
data.keys()[0], site.lang, path))
else:
wikipedia.output(u"Request %s:%s" % (site.lang, path))
@@ -95,7 +96,7 @@
params["User-agent"] = useragent
res = urllib2.urlopen(urllib2.Request(site.protocol() + '://'
+ site.hostname() + address, site.urlEncode(params)))
jsontext = res.read()
- elif params['action'] in postAC or data:
+ elif params['action'] in postAC:
res, jsontext = site.postForm(path, params, sysop,
site.cookies(sysop = sysop) )
else:
if back_response:
Modified: trunk/pywikipedia/redirect.py
===================================================================
--- trunk/pywikipedia/redirect.py 2009-10-03 04:52:39 UTC (rev 7359)
+++ trunk/pywikipedia/redirect.py 2009-10-03 08:54:22 UTC (rev 7360)
@@ -310,15 +310,11 @@
namespaces = [ 0 ]
#maxurllen = 1018 # accomodate "GET " + apiQ + CR + LF in 1024
bytes.
apiQ = []
- maxurllen = 900 - len(self.site.hostname() + self.site.api_address() )
- clen = 0
for pageid in self.get_redirect_pageids_via_api(number, namespaces,
start, until):
apiQ.append(pageid)
- clen += len(str(pageid)) + 1
- if clen > maxurllen and apiQ:
+ if len(apiQ) > 500:
yield apiQ
apiQ = []
- clen = 0
if apiQ:
yield apiQ
_______________________________________________
Pywikipedia-svn mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/pywikipedia-svn
