Xqt has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/789897 )
Change subject: [IMPR] Print counter statistic for all counters
......................................................................
[IMPR] Print counter statistic for all counters
BaseBot provides a counter which holds 'read', 'write' and 'skip'
by default but other counters can be added easily.
- rewrite exit() method that all counter values are printed
- increase 'read' counter before processing the page to add the 'read'
counter first. BotPage.counter keeps the order of insertion (with
Python 3.6+; the order of Python 3.5 is not deterministic)
- add tests accordingly
- add counters in touch.py when a page is touched or purged
- use 'upload' counter in specialbots/_upload.py
- Update BaseBot documentation
Bug: T307834
Change-Id: I567bae073e49eb3bde083b82a30e9f2a76044950
---
M docs/api_ref/pywikibot.rst
M pywikibot/bot.py
M pywikibot/specialbots/_upload.py
M scripts/touch.py
M tests/bot_tests.py
5 files changed, 80 insertions(+), 59 deletions(-)
Approvals:
Xqt: Verified; Looks good to me, approved
diff --git a/docs/api_ref/pywikibot.rst b/docs/api_ref/pywikibot.rst
index 02bd83a..cd8f47c 100644
--- a/docs/api_ref/pywikibot.rst
+++ b/docs/api_ref/pywikibot.rst
@@ -29,6 +29,7 @@
--------------------
.. automodule:: pywikibot.bot
+ :member-order: bysource
pywikibot.bot\_choice module
----------------------------
diff --git a/pywikibot/bot.py b/pywikibot/bot.py
index 274e4bf..e343643 100644
--- a/pywikibot/bot.py
+++ b/pywikibot/bot.py
@@ -1174,16 +1174,14 @@
'Option opt.bar is 4711'
"""
- # Handler configuration.
- # Only the keys of the dict can be passed as init options
- # The values are the default values
- # Overwrite this in subclasses!
-
available_options = {} # type: Dict[str, Any]
+ """ Handler configuration attribute.
+ Only the keys of the dict can be passed as `__init__` options.
+ The values are the default values. Overwrite this in subclasses!
+ """
def __init__(self, **kwargs: Any) -> None:
- """
- Only accept options defined in available_options.
+ """Only accept options defined in available_options.
:param kwargs: bot options
"""
@@ -1206,8 +1204,10 @@
class BaseBot(OptionHandler):
- """
- Generic Bot to be subclassed.
+ """Generic Bot to be subclassed.
+
+ Only accepts `generator` and options defined in
+ :attr:`available_options`.
This class provides a :meth:`run` method for basic processing of a
generator one page at a time.
@@ -1225,13 +1225,12 @@
properties.
If the subclass does not set a generator, or does not override
- :meth:`treat` or :meth:`run`, NotImplementedError is raised.
+ :meth:`treat` or :meth:`run`, `NotImplementedError` is raised.
For bot options handling refer :class:`OptionHandler` class above.
.. versionchanged:: 7.0
- A counter attribute is provided which is a `collections.Counter`;
- The default counters are 'read', 'write' and 'skip'.
+ A :attr:`counter` instance variable is provided.
"""
use_disambigs = None # type: Optional[bool]
@@ -1250,18 +1249,14 @@
.. versionadded:: 7.2
"""
- # Handler configuration.
- # The values are the default values
- # Extend this in subclasses!
-
available_options = {
'always': False, # By default ask for confirmation when putting a page
}
update_options = {} # type: Dict[str, Any]
- """update_options can be used to update available_options;
+ """`update_options` can be used to update :attr:`available_options`;
do not use it if the bot class is to be derived but use
- self.available_options.update(<dict>) initializer in such case.
+ `self.available_options.update(<dict>)` initializer in such case.
.. versionadded:: 6.4
"""
@@ -1269,7 +1264,7 @@
_current_page = None # type: Optional[pywikibot.page.BasePage]
def __init__(self, **kwargs: Any) -> None:
- """Only accept 'generator' and options defined in available_options.
+ """Initializer.
:param kwargs: bot options
:keyword generator: a :attr:`generator` processed by :meth:`run` method
@@ -1279,14 +1274,26 @@
pywikibot.warn('{} has a generator already. Ignoring argument.'
.format(self.__class__.__name__))
else:
- #: generator processed by :meth:`run` method
+ #: instance variable to hold the generator processed by
+ #: :meth:`run` method
self.generator = kwargs.pop('generator')
self.available_options.update(self.update_options)
super().__init__(**kwargs)
- self.counter = Counter()
+ self.counter = Counter() # type: Counter
+ """Instance variable which holds counters. The default counters
+ are 'read', 'write' and 'skip'. You can use your own counters like::
+
+ self.counter['delete'] += 1
+
+ .. versionadded:: 7.0
+ .. versionchanged:: 7.3
+ Your additional counters are also printed during :meth:`exit`
+ """
+
self._generator_completed = False
+
#: instance variable to hold the default page type
self.treat_page_type = pywikibot.page.BasePage # type: Any
@@ -1378,21 +1385,18 @@
"""
Save a new revision of a page, with user confirmation as required.
- Print differences, ask user for confirmation,
- and puts the page if needed.
+ Print differences, ask user for confirmation, and puts the page
+ if needed.
Option used:
* 'always'
- Keyword args used:
-
- * 'asynchronous' - passed to page.save
- * 'summary' - passed to page.save
- * 'show_diff' - show changes between oldtext and newtext (enabled)
- * 'ignore_save_related_errors' - report and ignore (disabled)
- * 'ignore_server_errors' - report and ignore (disabled)
-
+ :keyword asynchronous: passed to page.save
+ :keyword summary: passed to page.save
+ :keyword show_diff: show changes between oldtext and newtext (enabled)
+ :keyword ignore_save_related_errors: report and ignore (disabled)
+ :keyword ignore_server_errors: report and ignore (disabled)
:return: whether the page was saved successfully
"""
if oldtext.rstrip() == newtext.rstrip():
@@ -1403,7 +1407,6 @@
self.current_page = page
show_diff = kwargs.pop('show_diff', True)
-
if show_diff:
pywikibot.showDiff(oldtext, newtext)
@@ -1419,6 +1422,8 @@
"""
Helper function to handle page save-related option error handling.
+ .. note:: Do no use it directly. Use :meth:`userPut` instead.
+
:param page: currently edited page
:param func: the function to call
:param args: passed to the function
@@ -1430,6 +1435,8 @@
page save will be reported and ignored (default: False)
:kwtype ignore_save_related_errors: bool
:return: whether the page was saved successfully
+
+ :meta public:
"""
if not self.user_confirm('Do you want to accept these changes?'):
return False
@@ -1473,13 +1480,17 @@
raise QuitKeyboardInterrupt
def exit(self) -> None:
- """
- Cleanup and exit processing.
+ """Cleanup and exit processing.
- Invoked when Bot.run() is finished.
- Prints treat and save counters and informs whether the script
+ Invoked when :meth:`run` is finished. Waits for pending threads,
+ prints counter statistics and informs whether the script
terminated gracefully or was halted by exception.
- May be overridden by subclasses.
+
+ .. note:: Do not overwrite it by subclasses; :meth:`teardown`
+ should be used instead.
+
+ .. versionchanged:: 7.3
+ Statistics are printed for all entries in :attr:`counter`
"""
self.teardown()
if hasattr(self, '_start_ts'):
@@ -1489,10 +1500,10 @@
# wait until pending threads finished but don't close the queue
pywikibot.stopme()
- pywikibot.output('\n{read} pages read'
- '\n{write} pages written'
- '\n{skip} pages skipped'
- .format_map(self.counter))
+ pywikibot.info()
+ for op, count in self.counter.items():
+ pywikibot.info('{} {} operation{}'
+ .format(count, op, 's' if count > 1 else ''))
if hasattr(self, '_start_ts'):
write_delta = pywikibot.Timestamp.now() - self._start_ts
@@ -1508,10 +1519,12 @@
if self.counter['read']:
pywikibot.output('Read operation time: {:.1f} seconds'
.format(read_seconds / self.counter['read']))
- if self.counter['write']:
- pywikibot.output(
- 'Write operation time: {:.1f} seconds'
- .format(write_seconds / self.counter['write']))
+
+ for op, count in self.counter.items():
+ if not count or op == 'read':
+ continue
+ pywikibot.info('{} operation time: {:.1f} seconds'
+ .format(op.capitalize(), write_seconds / count))
# exc_info contains exception from self.run() while terminating
exc_info = sys.exc_info()
@@ -1525,13 +1538,15 @@
def init_page(self, item: Any) -> 'pywikibot.page.BasePage':
"""Initialize a generator item before treating.
- Ensure that the result of init_page is always a pywikibot.Page object
- even when the generator returns something else.
+ Ensure that the result of `init_page` is always a
+ pywikibot.Page object or any other type given by the
+ :attr:`treat_page_type` even when the generator returns
+ something else.
- Also used to set the arrange the current site. This is called before
- skip_page and treat.
+ Also used to set the arrange the current site. This is called
+ before :meth:`skip_page` and :meth:`treat`.
- :param item: any item from self.generator
+ :param item: any item from :attr:`generator`
:return: return the page object to be processed further
"""
return item
@@ -1576,17 +1591,18 @@
.format(self.__class__.__name__))
def setup(self) -> None:
- """Some initial setup before run operation starts.
+ """Some initial setup before :meth:`run` operation starts.
This can be used for reading huge parts from life wiki or file
operation which is more than just initialize the instance.
- Invoked by run() before running through generator loop.
+ Invoked by :meth:`run` before running through :attr:`generator`
+ loop.
.. versionadded:: 3.0
"""
def teardown(self) -> None:
- """Some cleanups after run operation. Invoked by exit().
+ """Some cleanups after :meth:`run` operation. Invoked by :meth:`exit`.
.. versionadded:: 3.0
"""
@@ -1621,8 +1637,8 @@
continue
# Process the page
- self.treat(page)
self.counter['read'] += 1
+ self.treat(page)
self._generator_completed = True
except QuitKeyboardInterrupt:
diff --git a/pywikibot/specialbots/_upload.py b/pywikibot/specialbots/_upload.py
index 52409eb..743f38e 100644
--- a/pywikibot/specialbots/_upload.py
+++ b/pywikibot/specialbots/_upload.py
@@ -431,7 +431,7 @@
# No warning, upload complete.
pywikibot.output('Upload of {} successful.'
.format(filename))
- self.counter['write'] += 1
+ self.counter['upload'] += 1
return filename # data['filename']
pywikibot.output('Upload aborted.')
break
diff --git a/scripts/touch.py b/scripts/touch.py
index 292f8ad..48386dd 100755
--- a/scripts/touch.py
+++ b/scripts/touch.py
@@ -61,6 +61,8 @@
.format(page.title(as_link=True)))
except PageSaveRelatedError as e:
pywikibot.error('Page {} not saved:\n{}'.format(page, e.args))
+ else:
+ self.counter['touch'] += 1
class PurgeBot(MultipleSitesBot):
@@ -76,9 +78,11 @@
def treat(self, page) -> None:
"""Purge the given page."""
+ done = page.purge(**self.opt)
+ if done:
+ self.counter['purge'] += 1
pywikibot.output('Page {}{} purged'
- .format(page,
- '' if page.purge(**self.opt) else ' not'))
+ .format(page, '' if done else ' not'))
def main(*args: str) -> None:
diff --git a/tests/bot_tests.py b/tests/bot_tests.py
index 12c89de..8b51092 100755
--- a/tests/bot_tests.py
+++ b/tests/bot_tests.py
@@ -195,7 +195,7 @@
pywikibot.Page(self.en, 'Page 2'),
pywikibot.Page(self.de, 'Page 3')],
post_treat)
- self.bot.exit = self._exit(2, exception=ValueError)
+ self.bot.exit = self._exit(3, exception=ValueError)
with self.assertRaisesRegex(ValueError, 'Whatever'):
self.bot.run()
@@ -212,7 +212,7 @@
pywikibot.Page(self.de, 'Page 3')],
post_treat)
- self.bot.exit = self._exit(2, exception=None)
+ self.bot.exit = self._exit(3, exception=None)
self.bot.run()
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/789897
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I567bae073e49eb3bde083b82a30e9f2a76044950
Gerrit-Change-Number: 789897
Gerrit-PatchSet: 6
Gerrit-Owner: Xqt <[email protected]>
Gerrit-Reviewer: D3r1ck01 <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
