Xqt has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/754076 )
Change subject: [bugfix] Enable default -mysqlquery with replace.py
......................................................................
[bugfix] Enable default -mysqlquery with replace.py
Default behaviour of -mysqlquery never worked with Pywikibot 3.0+
- use EXC_KEYS to handle exceptions
- move exceptions parsing to handle_exceptions()
- move pairsfile handling to handle_pairs()
- move manual handling to handle_manual()
- move sql handling to handle_sql()
- remove replacement_file_arg_misplaced check which is not necessary
if file_replacement is added after argument parsing
- shorten -xmlstart, -xml and -pairsfile using parsed value
- add local variable xmlStart to prevent NamingError
- only use sql_query and remove useSql variable
- disable parsing -mysqlquery with pagegenerators and enable local
option handling to enable the default dehaviour
- improvement for arg parsing
- decrease cyclomatic complexity by 22%
- update replacebot_tests.py
Bug: T299306
Change-Id: Iee845a1e0ab1bf1ab9c2f41349f59decb351f58b
---
M scripts/replace.py
M tests/replacebot_tests.py
2 files changed, 152 insertions(+), 125 deletions(-)
Approvals:
Xqt: Verified; Looks good to me, approved
diff --git a/scripts/replace.py b/scripts/replace.py
index 12cf5fe..5552141 100755
--- a/scripts/replace.py
+++ b/scripts/replace.py
@@ -146,9 +146,11 @@
import re
from collections.abc import Sequence
from contextlib import suppress
+from typing import Optional
import pywikibot
from pywikibot import editor, fixes, i18n, pagegenerators, textlib
+from pywikibot.backports import Dict, Generator, List, Pattern, Tuple
from pywikibot.bot import ExistingPageBot, SingleSiteBot
from pywikibot.exceptions import InvalidPageError, NoPageError
from pywikibot.tools import chars
@@ -768,6 +770,113 @@
return pattern
+EXC_KEYS = {
+ '-excepttitle': 'title',
+ '-requiretitle:': 'require-title',
+ '-excepttext': 'text-contains',
+ '-exceptinside': 'inside',
+ '-exceptinsidetag': 'inside-tags'
+}
+"""Dictionary to convert exceptions command line options to exceptions keys.
+
+ .. versionadded:: 7.0
+"""
+
+
+def handle_exceptions(*args: str) -> Tuple[List[str], Dict[str, str]]:
+ """Handle exceptions args to ignore pages which contain certain texts.
+
+ .. versionadded:: 7.0
+ """
+ exceptions = {key: [] for key in EXC_KEYS.values()}
+ local_args = []
+ for argument in args:
+ arg, _, value = argument.partition(':')
+ if arg in EXC_KEYS:
+ exceptions[EXC_KEYS[arg]].append(value)
+ else:
+ local_args.append(argument)
+ return local_args, exceptions
+
+
+def handle_pairsfile(filename: str) -> List[str]:
+ """Handle -pairsfile argument.
+
+ .. versionadded:: 7.0
+ """
+ if not filename:
+ filename = pywikibot.input(
+ 'Please enter the filename to read replacements from:')
+
+ try:
+ with codecs.open(filename, 'r', 'utf-8') as f:
+ # strip newlines, but not other characters
+ replacements = f.read().splitlines()
+ if not replacements:
+ raise OSError('{} is empty.'.format(filename))
+ except OSError as e:
+ pywikibot.error('Error loading {}: {}'.format(filename, e))
+ return None
+
+ if len(replacements) % 2:
+ pywikibot.error(
+ '{} contains an incomplete pattern replacement pair.'.format(
+ filename))
+ return None
+
+ # Strip BOM from first line
+ replacements[0].lstrip('\uFEFF')
+ return replacements
+
+
+def handle_manual() -> List[str]:
+ """Handle manual input.
+
+ .. versionadded:: 7.0
+ """
+ pairs = []
+ old = pywikibot.input('Please enter the text that should be replaced:')
+ while old:
+ new = pywikibot.input('Please enter the new text:')
+ pairs += [old, new]
+ old = pywikibot.input(
+ 'Please enter another text that should be replaced,\n'
+ 'or press Enter to start:')
+ return pairs
+
+
+def handle_sql(sql: str,
+ replacements: List[Pattern],
+ exceptions: List[Pattern]) -> Generator:
+ """Handle default sql query.
+
+ .. versionadded:: 7.0
+ """
+ if not sql:
+ where_clause = 'WHERE ({})'.format(' OR '.join(
+ "old_text RLIKE '{}'"
+ .format(prepareRegexForMySQL(repl.old_regex.pattern))
+ for repl in replacements))
+
+ if exceptions:
+ except_clause = 'AND NOT ({})'.format(' OR '.join(
+ "old_text RLIKE '{}'"
+ .format(prepareRegexForMySQL(exc.pattern))
+ for exc in exceptions))
+ else:
+ except_clause = ''
+
+ sql = """
+SELECT page_namespace, page_title
+FROM page
+JOIN text ON (page_id = old_id)
+{}
+{}
+LIMIT 200""".format(where_clause, except_clause)
+
+ return pagegenerators.MySQLPageGenerator(sql)
+
+
def main(*args: str) -> None:
"""
Process command line arguments and invoke bot.
@@ -783,16 +892,9 @@
# Array which will collect commandline parameters.
# First element is original text, second element is replacement text.
commandline_replacements = []
+ file_replacements = []
# A list of 2-tuples of original text and replacement text.
replacements = []
- # Don't edit pages which contain certain texts.
- exceptions = {
- 'title': [],
- 'text-contains': [],
- 'inside': [],
- 'inside-tags': [],
- 'require-title': [], # using a separate requirements dict needs some
- } # major refactoring of code.
# Should the elements of 'replacements' and 'exceptions' be interpreted
# as regular expressions?
@@ -802,53 +904,36 @@
# the dump's path, either absolute or relative, which will be used
# if -xml flag is present
xmlFilename = None
- useSql = False
- sql_query = None
+ xmlStart = None
+ sql_query = None # type: Optional[str]
# Set the default regular expression flags
flags = 0
# Request manual replacements even if replacements are already defined
manual_input = False
- # Replacements loaded from a file
- replacement_file = None
- replacement_file_arg_misplaced = False
# Read commandline parameters.
+ genFactory = pagegenerators.GeneratorFactory(
+ disabled_options=['mysqlquery'])
local_args = pywikibot.handle_args(args)
- genFactory = pagegenerators.GeneratorFactory()
local_args = genFactory.handle_args(local_args)
+ local_args, exceptions = handle_exceptions(*local_args)
- for arg in local_args:
+ for argument in local_args:
+ arg, _, value = argument.partition(':')
if arg == '-regex':
regex = True
- elif arg.startswith('-xmlstart'):
- if len(arg) == 9:
- xmlStart = pywikibot.input(
- 'Please enter the dumped article to start with:')
- else:
- xmlStart = arg[10:]
- elif arg.startswith('-xml'):
- if len(arg) == 4:
- xmlFilename = i18n.input('pywikibot-enter-xml-filename')
- else:
- xmlFilename = arg[5:]
- elif arg.startswith('-mysqlquery'):
- useSql = True
- sql_query = arg.partition(':')[2]
- elif arg.startswith('-excepttitle:'):
- exceptions['title'].append(arg[13:])
- elif arg.startswith('-requiretitle:'):
- exceptions['require-title'].append(arg[14:])
- elif arg.startswith('-excepttext:'):
- exceptions['text-contains'].append(arg[12:])
- elif arg.startswith('-exceptinside:'):
- exceptions['inside'].append(arg[14:])
- elif arg.startswith('-exceptinsidetag:'):
- exceptions['inside-tags'].append(arg[17:])
- elif arg.startswith('-fix:'):
- fixes_set += [arg[5:]]
- elif arg.startswith('-sleep:'):
- options['sleep'] = float(arg[7:])
- elif arg in ('-always', '-recursive', '-allowoverlap'):
+ elif arg == '-xmlstart':
+ xmlStart = value or pywikibot.input(
+ 'Please enter the dumped article to start with:')
+ elif arg == '-xml':
+ xmlFilename = value or i18n.input('pywikibot-enter-xml-filename')
+ elif arg == '-mysqlquery':
+ sql_query = value
+ elif arg == '-fix':
+ fixes_set.append(value)
+ elif arg == '-sleep':
+ options['sleep'] = float(value)
+ elif arg in ('-allowoverlap', '-always', '-recursive'):
options[arg[1:]] = True
elif arg == '-nocase':
flags |= re.IGNORECASE
@@ -856,67 +941,32 @@
flags |= re.DOTALL
elif arg == '-multiline':
flags |= re.MULTILINE
- elif arg.startswith('-addcat:'):
- options['addcat'] = arg[8:]
- elif arg.startswith('-summary:'):
- edit_summary = arg[9:]
- elif arg.startswith('-automaticsummary'):
+ elif arg == '-addcat':
+ options['addcat'] = value
+ elif arg == '-summary':
+ edit_summary = value
+ elif arg == '-automaticsummary':
edit_summary = True
- elif arg.startswith('-manualinput'):
+ elif arg == '-manualinput':
manual_input = True
- elif arg.startswith('-pairsfile'):
- if len(commandline_replacements) % 2:
- replacement_file_arg_misplaced = True
-
- if arg == '-pairsfile':
- replacement_file = pywikibot.input(
- 'Please enter the filename to read replacements from:')
- else:
- replacement_file = arg[len('-pairsfile:'):]
+ elif arg == '-pairsfile':
+ file_replacements = handle_pairsfile(value)
else:
commandline_replacements.append(arg)
- site = pywikibot.Site()
+ if file_replacements is None:
+ return
if len(commandline_replacements) % 2:
pywikibot.error('Incomplete command line pattern replacement pair.')
return
- if replacement_file_arg_misplaced:
- pywikibot.error(
- '-pairsfile used between a pattern replacement pair.')
- return
-
- if replacement_file:
- try:
- with codecs.open(replacement_file, 'r', 'utf-8') as f:
- # strip newlines, but not other characters
- file_replacements = f.read().splitlines()
- except OSError as e:
- pywikibot.error('Error loading {}: {}'.format(
- replacement_file, e))
- return
-
- if len(file_replacements) % 2:
- pywikibot.error(
- '{} contains an incomplete pattern replacement pair.'.format(
- replacement_file))
- return
-
- # Strip BOM from first line
- file_replacements[0].lstrip('\uFEFF')
- commandline_replacements.extend(file_replacements)
-
+ commandline_replacements += file_replacements
if not(commandline_replacements or fixes_set) or manual_input:
- old = pywikibot.input('Please enter the text that should be replaced:')
- while old:
- new = pywikibot.input('Please enter the new text:')
- commandline_replacements += [old, new]
- old = pywikibot.input(
- 'Please enter another text that should be replaced,'
- '\nor press Enter to start:')
+ commandline_replacements += handle_manual()
# The summary stored here won't be actually used but is only an example
+ site = pywikibot.Site()
single_summary = None
for i in range(0, len(commandline_replacements), 2):
replacement = Replacement(commandline_replacements[i],
@@ -1029,33 +1079,12 @@
precompile_exceptions(exceptions, regex, flags)
if xmlFilename:
- try:
- xmlStart
- except NameError:
- xmlStart = None
gen = XmlDumpReplacePageGenerator(xmlFilename, xmlStart,
replacements, exceptions, site)
- elif useSql:
- if not sql_query:
- whereClause = 'WHERE ({})'.format(' OR '.join(
- "old_text RLIKE '{}'"
- .format(prepareRegexForMySQL(old_regexp.pattern))
- for (old_regexp, new_text) in replacements))
- if exceptions:
- exceptClause = 'AND NOT ({})'.format(' OR '.join(
- "old_text RLIKE '{}'"
- .format(prepareRegexForMySQL(exc.pattern))
- for exc in exceptions))
- else:
- exceptClause = ''
- query = sql_query or """
-SELECT page_namespace, page_title
-FROM page
-JOIN text ON (page_id = old_id)
-{}
-{}
-LIMIT 200""".format(whereClause, exceptClause)
- gen = pagegenerators.MySQLPageGenerator(query)
+ elif sql_query is not None:
+ # Only -excepttext option is considered by the query. Other
+ # exceptions are taken into account by the ReplaceRobot
+ gen = handle_sql(sql_query, replacements, exceptions['text-contains'])
gen = genFactory.getCombinedGenerator(gen, preload=True)
if pywikibot.bot.suggest_help(missing_generator=not gen):
diff --git a/tests/replacebot_tests.py b/tests/replacebot_tests.py
index 9ee153b..a4ac228 100644
--- a/tests/replacebot_tests.py
+++ b/tests/replacebot_tests.py
@@ -100,16 +100,14 @@
def test_invalid_replacements(self):
"""Test invalid command line replacement configurations."""
- # old and new need to be together
- self.assertFalse(self._run('foo', '-pairsfile:/dev/null', 'bar'))
-
- self.assertEqual([
- '-pairsfile used between a pattern replacement pair.',
- ], pywikibot.bot.ui.pop_output())
+ # old and new no longer need to be together but pairsfile must exist
+ self._run('foo', '-pairsfile:/dev/null', 'bar')
+ self.assertIn('Error loading /dev/null:',
+ pywikibot.bot.ui.pop_output()[0])
# only old provided
with empty_sites():
- self.assertFalse(self._run('foo'))
+ self._run('foo')
self.assertEqual([
'Incomplete command line pattern replacement pair.',
], pywikibot.bot.ui.pop_output())
@@ -146,7 +144,7 @@
def _get_bot(self, only_confirmation, *args):
"""Run with arguments, assert and return one bot."""
- self.assertIsNone(self._run(*args))
+ self._run(*args)
self.assertLength(self.bots, 1)
bot = self.bots[0]
if only_confirmation is not None:
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/754076
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Iee845a1e0ab1bf1ab9c2f41349f59decb351f58b
Gerrit-Change-Number: 754076
Gerrit-PatchSet: 14
Gerrit-Owner: Xqt <[email protected]>
Gerrit-Reviewer: D3r1ck01 <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
