Matěj Suchánek has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/364475 )
Change subject: [IMPR] Enhance argument management in harvest_template.py
......................................................................
[IMPR] Enhance argument management in harvest_template.py
Now each param-property pair holds its OptionHandler where
options specific for this claim are stored. Additionally,
it's possible to apply an option to the bot which applies
the option to all claims. Each claim can override this option
if the input is correct (see the updated instructions).
The only setting is at the moment -islink, which makes the
feature added in I71f6857ca optional.
Bug: T64014
Bug: T72702
Change-Id: Ibc4a0f845fe88349d49b787b5f16b20eb650ef8e
---
M scripts/harvest_template.py
1 file changed, 110 insertions(+), 19 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core
refs/changes/75/364475/1
diff --git a/scripts/harvest_template.py b/scripts/harvest_template.py
index b9954af..538d412 100755
--- a/scripts/harvest_template.py
+++ b/scripts/harvest_template.py
@@ -6,9 +6,13 @@
Usage:
* python pwb.py harvest_template -transcludes:"..." \
- template_parameter PID [template_parameter PID]
+ [default optional arguments] \
+ template_parameter PID [local optional arguments] \
+ [template_parameter PID [local optional arguments]]
* python pwb.py harvest_template [generators] -template:"..." \
- template_parameter PID [template_parameter PID]
+ [default optional arguments] \
+ template_parameter PID [local optional arguments] \
+ [template_parameter PID [local optional arguments]]
This will work on all pages that transclude the template in the article
namespace
@@ -16,6 +20,14 @@
These command line parameters can be used to specify which pages to work on:
¶ms;
+
+The following command line parameters can be used to change the bot's behavior.
+If you specify them before all parameters, they are global and are applied to
+all param-property pairs. If you specify them after a param-property pair,
+they are local and are only applied to this pair. If you specify the same
+argument as both local and global, the local argument overrides the global one.
+
+-islink Treat plain text values as links ("text" -> "[[text]]").
Examples:
@@ -52,16 +64,27 @@
signal.signal(signal.SIGINT, _signal_handler)
import pywikibot
-from pywikibot import pagegenerators as pg, WikidataBot
+from pywikibot import pagegenerators as pg, WikidataBot, OptionHandler
docuReplacements = {'¶ms;': pywikibot.pagegenerators.parameterHelp}
+
+
+class PropertyOptionHandler(OptionHandler):
+
+ """
+ Class holding options for a param-property pair.
+ """
+
+ availableOptions = {
+ 'islink': False,
+ }
class HarvestRobot(WikidataBot):
"""A bot to add Wikidata claims."""
- def __init__(self, generator, templateTitle, fields):
+ def __init__(self, generator, templateTitle, fields, **kwargs):
"""
Constructor.
@@ -71,9 +94,14 @@
@type templateTitle: str
@param fields: A dictionary of fields that are of use to us
@type fields: dict
+ @keyword islink: Whether non-linked values should be treated as links
+ @type islink: bool
"""
- self.availableOptions['always'] = True
- super(HarvestRobot, self).__init__()
+ self.availableOptions.update({
+ 'always': True,
+ 'islink': False,
+ })
+ super(HarvestRobot, self).__init__(**kwargs)
self.generator = generator
self.templateTitle = templateTitle.replace(u'_', u' ')
# TODO: Make it a list which also includes the redirects to the
template
@@ -127,13 +155,35 @@
return linked_item
+ def _get_option_with_fallback(self, handler, option):
+ """
+ +---------+-------+--------+
+ | default | local | result |
+ +=========+=======+========+
+ | False | False | False |
+ +---------+-------+--------+
+ | False | True | True |
+ +---------+-------+--------+
+ | True | False | True |
+ +--------------------------+
+ | True | True | False |
+ +--------------------------+
+
+ TODO: only works with booleans
+ """
+ default = self.getOption(option)
+ local = handler.getOption(option)
+ return default is not local
+
def treat(self, page, item):
"""Process a single page/item."""
if willstop:
raise KeyboardInterrupt
self.current_page = page
item.get()
- if set(self.fields.values()) <= set(item.claims.keys()):
+ # b/c
+ if set(map(lambda val: val[0] if isinstance(value, tuple) else val
+ self.fields.values()) <= set(item.claims.keys()):
pywikibot.output('%s item %s has claims for all properties. '
'Skipping.' % (page, item.title()))
return
@@ -159,8 +209,14 @@
# This field contains something useful for us
if field in self.fields:
+ # b/c
+ if isinstance(self.fields[field], tuple):
+ prop, options = self.fields[field]
+ else:
+ prop = self.fields[field]
+ options = PropertyOptionHandler()
# Check if the property isn't already set
- claim = pywikibot.Claim(self.repo, self.fields[field])
+ claim = pywikibot.Claim(self.repo, prop)
if claim.getID() in item.get().get('claims'):
pywikibot.output(
'A claim for %s already exists. Skipping.'
@@ -175,7 +231,15 @@
if match:
link_text = match.group(1)
else:
- link_text = value
+ if self._get_option_with_fallback(
+ options, 'islink'):
+ link_text = value
+ else:
+ pywikibot.output(
+ '%s field %s value %s is not a '
+ 'wikilink. Skipping.'
+ % (claim.getID(), field, value))
+ continue
linked_item = self._template_link_target(
item, link_text)
@@ -220,13 +284,15 @@
@param args: command line arguments
@type args: list of unicode
"""
- commandline_arguments = []
- template_title = u''
+ template_title = None
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
gen = pg.GeneratorFactory()
+ current_args = []
+ fields = {}
+ options = {}
for arg in local_args:
if arg.startswith('-template'):
if len(arg) == 9:
@@ -238,19 +304,44 @@
if arg.startswith(u'-transcludes:'):
template_title = arg[13:]
else:
- commandline_arguments.append(arg)
+ optional = arg.startswith('-')
+ complete = len(current_args) == 3
+ if optional:
+ needs_second = len(current_args) == 1
+ if needs_second:
+ break # will stop below
+
+ arg, sep, value = arg[1:].partition(':')
+ if len(current_args) == 0:
+ assert not fields
+ options[arg] = value if value != '' else True
+ else:
+ assert complete
+ current_args[2].update({
+ arg: value if value != '' else True
+ })
+ else:
+ if complete:
+ handler = PropertyOptionHandler(**current_args[2])
+ fields[current_args[0]] = (current_args[1], handler)
+ current_args.clear()
+ else:
+ current_args.append(arg)
+ if len(current_args) == 2:
+ current_args.append({})
+
+ # handle left over
+ if len(current_args) == 3:
+ handler = PropertyOptionHandler(**current_args[2])
+ fields[current_args[0]] = (current_args[1], handler)
+ elif len(current_args) == 1:
+ pywikibot.error('Incomplete command line param-property pair.')
+ return False
if not template_title:
pywikibot.error(
'Please specify either -template or -transcludes argument')
return
-
- if len(commandline_arguments) % 2:
- raise ValueError # or something.
- fields = {}
-
- for i in range(0, len(commandline_arguments), 2):
- fields[commandline_arguments[i]] = commandline_arguments[i + 1]
generator = gen.getCombinedGenerator(preload=True)
if not generator:
--
To view, visit https://gerrit.wikimedia.org/r/364475
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibc4a0f845fe88349d49b787b5f16b20eb650ef8e
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Matěj Suchánek <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
