Whym has uploaded a new change for review.
https://gerrit.wikimedia.org/r/321862
Change subject: timestripper: prevent recognizing components far from each other
......................................................................
timestripper: prevent recognizing components far from each other
timestripper should not be too flexible about the locations
of the components of a timestamp. The added test demonstrates
a false positive, incorrectly recognized as a timestamp.
This patch places a limit to the distance between the left most
and right most components. Tentatively the limit is set to 50.
Change-Id: I8ef86e21f08248d6abb7d1b78252029d2ce0c017
---
M pywikibot/textlib.py
M tests/timestripper_tests.py
2 files changed, 25 insertions(+), 4 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core
refs/changes/62/321862/1
diff --git a/pywikibot/textlib.py b/pywikibot/textlib.py
index fc7c4b1..0503680 100644
--- a/pywikibot/textlib.py
+++ b/pywikibot/textlib.py
@@ -128,6 +128,8 @@
'or': u'୦୧୨୩୪୫୬୭୮୯',
}
+TIMESTAMP_LENGTH_LIMIT = 50
+
def to_local_digits(phrase, lang):
"""
@@ -1972,13 +1974,18 @@
return (txt, None)
@staticmethod
- def _valid_date_dict_order(dateDict):
+ def _valid_date_dict_positions(dateDict):
"""Check consistency of reasonable positions for groups."""
day_pos = dateDict['day']['pos']
month_pos = dateDict['month']['pos']
year_pos = dateDict['year']['pos']
time_pos = dateDict['time']['pos']
tzinfo_pos = dateDict['tzinfo']['pos']
+
+ if max(day_pos, month_pos, year_pos, time_pos, tzinfo_pos) \
+ - min(day_pos, month_pos, year_pos, time_pos, tzinfo_pos) \
+ > TIMESTAMP_LENGTH_LIMIT:
+ return False
date_pos = sorted((day_pos, month_pos, year_pos))
min_pos, max_pos = date_pos[0], date_pos[-1]
@@ -2031,7 +2038,7 @@
# all fields matched -> date valid
# groups are in a reasonable order.
if (all(g in dateDict for g in self.groups) and
- self._valid_date_dict_order(dateDict)):
+ self._valid_date_dict_positions(dateDict)):
# remove 'time' key, now split in hour/minute and not needed
# by datetime.
del dateDict['time']
diff --git a/tests/timestripper_tests.py b/tests/timestripper_tests.py
index 90eca23..f695e8e 100644
--- a/tests/timestripper_tests.py
+++ b/tests/timestripper_tests.py
@@ -99,8 +99,8 @@
txtHourInRange = u'7 février 2010 à 23:00 (CET)'
txtHourOutOfRange = u'7 février 2010 à 24:00 (CET)'
- self.assertNotEqual(self.ts.timestripper(txtHourInRange), None)
- self.assertEqual(self.ts.timestripper(txtHourOutOfRange), None)
+ self.assertIsNotNone(self.ts.timestripper(txtHourInRange))
+ self.assertIsNone(self.ts.timestripper(txtHourOutOfRange))
class TestTimeStripperWithDigitsAsMonths(TestTimeStripperCase):
@@ -142,6 +142,20 @@
)
+class TestTimeStripperIllFormed(TestTimeStripperCase):
+
+ """Test cases for rejecting timestamp-like texts that are not
timestamps."""
+
+ family = 'wikipedia'
+ code = 'en'
+
+ def test_missing_year(self):
+ """Test that year is required for a complete timestamp."""
+ self.assertIsNone(self.ts.timestripper(
+ '2000 people will attend and it will start on 16 December at 22:00
(UTC).'
+ ))
+
+
class TestTimeStripperLanguage(TestCase):
"""Test cases for English language."""
--
To view, visit https://gerrit.wikimedia.org/r/321862
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8ef86e21f08248d6abb7d1b78252029d2ce0c017
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Whym <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
