Mpaa created this task. Mpaa added a project: Pywikibot. Restricted Application added subscribers: pywikibot-bugs-list, Aklapper.
TASK DESCRIPTION See https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.iterparse > Note > > iterparse() only guarantees that it has seen the “>” character of a starting tag when it emits a “start” event, so the attributes are defined, but the contents of the text and tail attributes are undefined at that point. The same applies to the element children; they may or may not be present. > > If you need a fully populated element, look for “end” events instead. **Steps to replicate the issue** (include links if applicable): - apply this patch diff --git a/pywikibot/xmlreader.py b/pywikibot/xmlreader.py index 5d3575e6b..98a420364 100644 --- a/pywikibot/xmlreader.py +++ b/pywikibot/xmlreader.py @@ -235,10 +235,23 @@ class XmlDump: elem.clear() self.root.clear() + def _assert(self, elem): + "assert." + assert self.title == elem.findtext(f'{{{self.uri}}}title') + assert self.ns == elem.findtext(f'{{{self.uri}}}ns') + assert self.pageid == elem.findtext(f'{{{self.uri}}}id') + assert self.restrictions == elem.findtext(f'{{{self.uri}}}restrictions') + assert self.isredirect == (elem.findtext(f'{{{self.uri}}}redirect') is not None) + edit, move = parseRestrictions(elem.findtext(f'{{{self.uri}}}restrictions')) + assert self.editRestriction == edit + assert self.moveRestriction == move + def _parse_all(self, event, elem): """Parser that yields all revisions.""" if event == 'start' and elem.tag == f'{{{self.uri}}}page': self._headers(elem) + if event == 'end' and elem.tag == f'{{{self.uri}}}page': + self._assert(elem) if event == 'end' and elem.tag == f'{{{self.uri}}}revision': yield self._create_revision(elem) elem.clear() - run from pywikibot import xmlreader filename = 'enwikisource.xml'# use a dump with full history per page dump = xmlreader.XmlDump(filename, allrevisions=True) gen = dump.parse() for i, rev in enumerate(gen): rev.text = rev.text[0:10] + ' ...' print((rev.__dict__['revisionid'], vars(rev))) - **What happens?**: ('4054131', {'title': 'Presidential Radio Address - 3 March 2001', 'ns': '0', 'id': '', 'text': '{{header\n ...', 'username': 'Mpaa', 'ipedit': False, 'timestamp': '2012-09-06T23:31:28Z', 'editRestriction': None, 'moveRestriction': None, 'revisionid': '4054131', 'comment': 'year/noyearcat in header + changed cat + changed note', 'isredirect': False}) --------------------------------------------------------------------------- AssertionError Traceback (most recent call last) File ~/Downloads/xmldump/test_xml.py:63 42 yield next(dump.parse()) 45 #import tracemalloc 46 #tracemalloc.start() 47 #print(tracemalloc.get_traced_memory()) (...) 60 #print(tracemalloc.get_traced_memory()) 61 #tracemalloc.stop() ---> 63 for i, rev in enumerate(gen): 64 rev.text = rev.text[0:10] + ' ...' 65 print((rev.__dict__['revisionid'], vars(rev))) File ~/python/core/pywikibot/xmlreader.py:190, in XmlDump.parse(self) 188 self.root = elem 189 continue --> 190 yield from self._parse(event, elem) File ~/python/core/pywikibot/xmlreader.py:254, in XmlDump._parse_all(self, event, elem) 252 self._headers(elem) 253 if event == 'end' and elem.tag == f'{{{self.uri}}}page': --> 254 self._assert(elem) 255 if event == 'end' and elem.tag == f'{{{self.uri}}}revision': 256 yield self._create_revision(elem) File ~/python/core/pywikibot/xmlreader.py:242, in XmlDump._assert(self, elem) 240 assert self.title == elem.findtext(f'{{{self.uri}}}title') 241 assert self.ns == elem.findtext(f'{{{self.uri}}}ns') --> 242 assert self.pageid == elem.findtext(f'{{{self.uri}}}id') 243 assert self.restrictions == elem.findtext(f'{{{self.uri}}}restrictions') 244 assert self.isredirect == (elem.findtext(f'{{{self.uri}}}redirect') is not None) AssertionError: > /home/squiddy/python/core/pywikibot/xmlreader.py(242)_assert() 240 assert self.title == elem.findtext(f'{{{self.uri}}}title') 241 assert self.ns == elem.findtext(f'{{{self.uri}}}ns') --> 242 assert self.pageid == elem.findtext(f'{{{self.uri}}}id') 243 assert self.restrictions == elem.findtext(f'{{{self.uri}}}restrictions') 244 assert self.isredirect == (elem.findtext(f'{{{self.uri}}}redirect') is not None) ipdb> elem.findtext(f'{{{self.uri}}}id') '317' **What should have happened instead?**: no errors TASK DETAIL https://phabricator.wikimedia.org/T354095 EMAIL PREFERENCES https://phabricator.wikimedia.org/settings/panel/emailpreferences/ To: Mpaa Cc: Aklapper, pywikibot-bugs-list, Mpaa, mevo, PotsdamLamb, Jyoo1011, JohnsonLee01, SHEKH, Dijkstra, Khutuck, Zkhalido, Viztor, Wenyi, Tbscho, MayS, Mdupont, JJMC89, Dvorapa, Altostratus, Avicennasis, mys_721tx, Xqt, jayvdb, Masti, Alchimista
_______________________________________________ pywikibot-bugs mailing list -- [email protected] To unsubscribe send an email to [email protected]
