Serhiy Storchaka added the comment:
Here are complete patch and script used to generate equivalence table.
----------
Added file: http://bugs.python.org/file37086/re_ignore_case_2.patch
Added file: http://bugs.python.org/file37087/re_cases.py
_______________________________________
Python tracker <rep...@bugs.python.org>
<http://bugs.python.org/issue12728>
_______________________________________
diff -r 79eb4ab497b1 Lib/sre_compile.py
--- a/Lib/sre_compile.py Fri Oct 31 12:40:11 2014 +0200
+++ b/Lib/sre_compile.py Fri Oct 31 17:59:04 2014 +0200
@@ -27,6 +27,46 @@ else:
_SUCCESS_CODES = set([SUCCESS, FAILURE])
_ASSERT_CODES = set([ASSERT, ASSERT_NOT])
+# Sets of characters which have the same uppercase but different lowercase.
+_equivalences = (
+ # LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I
+ (0x69, 0x131), # iı
+ # LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S
+ (0x73, 0x17f), # sſ
+ # MICRO SIGN, GREEK SMALL LETTER MU
+ (0xb5, 0x3bc), # µμ
+ # COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK
PROSGEGRAMMENI
+ (0x345, 0x3b9, 0x1fbe), # \u0345ιι
+ # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER
IOTA WITH DIALYTIKA AND OXIA
+ (0x390, 0x1fd3), # Îá¿
+ # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER
UPSILON WITH DIALYTIKA AND OXIA
+ (0x3b0, 0x1fe3), # ΰΰ
+ # GREEK SMALL LETTER BETA, GREEK BETA SYMBOL
+ (0x3b2, 0x3d0), # βÏ
+ # GREEK SMALL LETTER EPSILON, GREEK LUNATE EPSILON SYMBOL
+ (0x3b5, 0x3f5), # εϵ
+ # GREEK SMALL LETTER THETA, GREEK THETA SYMBOL
+ (0x3b8, 0x3d1), # θÏ
+ # GREEK SMALL LETTER KAPPA, GREEK KAPPA SYMBOL
+ (0x3ba, 0x3f0), # κϰ
+ # GREEK SMALL LETTER PI, GREEK PI SYMBOL
+ (0x3c0, 0x3d6), # ÏÏ
+ # GREEK SMALL LETTER RHO, GREEK RHO SYMBOL
+ (0x3c1, 0x3f1), # Ïϱ
+ # GREEK SMALL LETTER FINAL SIGMA, GREEK SMALL LETTER SIGMA
+ (0x3c2, 0x3c3), # ÏÏ
+ # GREEK SMALL LETTER PHI, GREEK PHI SYMBOL
+ (0x3c6, 0x3d5), # ÏÏ
+ # LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT
ABOVE
+ (0x1e61, 0x1e9b), # ṡáº
+ # LATIN SMALL LIGATURE LONG S T, LATIN SMALL LIGATURE ST
+ (0xfb05, 0xfb06), # ï¬
ï¬
+)
+
+# Mapping from lowercase code to lowercase codes which have the same uppercase.
+_ignorecase_fixes = {i: tuple(j for j in t if i != j)
+ for t in _equivalences for i in t}
+
def _compile(code, pattern, flags):
# internal: compile a (sub)pattern
emit = code.append
@@ -35,11 +75,29 @@ def _compile(code, pattern, flags):
REPEATING_CODES = _REPEATING_CODES
SUCCESS_CODES = _SUCCESS_CODES
ASSERT_CODES = _ASSERT_CODES
+ if (flags & SRE_FLAG_IGNORECASE and
+ not (flags & SRE_FLAG_LOCALE) and
+ flags & SRE_FLAG_UNICODE):
+ fixes = _ignorecase_fixes
+ else:
+ fixes = None
for op, av in pattern:
if op in LITERAL_CODES:
if flags & SRE_FLAG_IGNORECASE:
- emit(OPCODES[OP_IGNORE[op]])
- emit(_sre.getlower(av, flags))
+ lo = _sre.getlower(av, flags)
+ if fixes and lo in fixes:
+ emit(OPCODES[IN_IGNORE])
+ skip = _len(code); emit(0)
+ if op is NOT_LITERAL:
+ emit(OPCODES[NEGATE])
+ for k in (lo,) + fixes[lo]:
+ emit(OPCODES[LITERAL])
+ emit(k)
+ emit(OPCODES[FAILURE])
+ code[skip] = _len(code) - skip
+ else:
+ emit(OPCODES[OP_IGNORE[op]])
+ emit(lo)
else:
emit(OPCODES[op])
emit(av)
@@ -52,7 +110,7 @@ def _compile(code, pattern, flags):
emit(OPCODES[op])
fixup = None
skip = _len(code); emit(0)
- _compile_charset(av, flags, code, fixup)
+ _compile_charset(av, flags, code, fixup, fixes)
code[skip] = _len(code) - skip
elif op is ANY:
if flags & SRE_FLAG_DOTALL:
@@ -166,10 +224,10 @@ def _compile(code, pattern, flags):
else:
raise ValueError("unsupported operand type", op)
-def _compile_charset(charset, flags, code, fixup=None):
+def _compile_charset(charset, flags, code, fixup=None, fixes=None):
# compile charset subprogram
emit = code.append
- for op, av in _optimize_charset(charset, fixup):
+ for op, av in _optimize_charset(charset, fixup, fixes):
emit(OPCODES[op])
if op is NEGATE:
pass
@@ -193,7 +251,7 @@ def _compile_charset(charset, flags, cod
raise error("internal: unsupported set operator")
emit(OPCODES[FAILURE])
-def _optimize_charset(charset, fixup):
+def _optimize_charset(charset, fixup, fixes):
# internal: optimize character set
out = []
tail = []
@@ -203,14 +261,26 @@ def _optimize_charset(charset, fixup):
try:
if op is LITERAL:
if fixup:
- av = fixup(av)
- charmap[av] = 1
+ lo = fixup(av)
+ charmap[lo] = 1
+ if fixes and lo in fixes:
+ for k in fixes[lo]:
+ charmap[k] = 1
+ else:
+ charmap[av] = 1
elif op is RANGE:
r = range(av[0], av[1]+1)
if fixup:
r = map(fixup, r)
- for i in r:
- charmap[i] = 1
+ if fixup and fixes:
+ for i in r:
+ charmap[i] = 1
+ if i in fixes:
+ for k in fixes[i]:
+ charmap[k] = 1
+ else:
+ for i in r:
+ charmap[i] = 1
elif op is NEGATE:
out.append((op, av))
else:
diff -r 79eb4ab497b1 Lib/test/test_re.py
--- a/Lib/test/test_re.py Fri Oct 31 12:40:11 2014 +0200
+++ b/Lib/test/test_re.py Fri Oct 31 17:59:04 2014 +0200
@@ -601,6 +601,43 @@ class ReTests(unittest.TestCase):
self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a
a")
self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1),
"a aa")
+ assert '\u212a'.lower() == 'k' # 'âª'
+ self.assertTrue(re.match(r'K', '\u212a', re.I))
+ self.assertTrue(re.match(r'k', '\u212a', re.I))
+ self.assertTrue(re.match(r'\u212a', 'K', re.I))
+ self.assertTrue(re.match(r'\u212a', 'k', re.I))
+ assert '\u017f'.upper() == 'S' # 'Å¿'
+ self.assertTrue(re.match(r'S', '\u017f', re.I))
+ self.assertTrue(re.match(r's', '\u017f', re.I))
+ self.assertTrue(re.match(r'\u017f', 'S', re.I))
+ self.assertTrue(re.match(r'\u017f', 's', re.I))
+ assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ï¬
', 'ï¬'
+ self.assertTrue(re.match(r'\ufb05', '\ufb06', re.I))
+ self.assertTrue(re.match(r'\ufb06', '\ufb05', re.I))
+
+ def test_ignore_case_set(self):
+ self.assertTrue(re.match(r'[19A]', 'A', re.I))
+ self.assertTrue(re.match(r'[19a]', 'a', re.I))
+ self.assertTrue(re.match(r'[19a]', 'A', re.I))
+ self.assertTrue(re.match(r'[19A]', 'a', re.I))
+ self.assertTrue(re.match(br'[19A]', b'A', re.I))
+ self.assertTrue(re.match(br'[19a]', b'a', re.I))
+ self.assertTrue(re.match(br'[19a]', b'A', re.I))
+ self.assertTrue(re.match(br'[19A]', b'a', re.I))
+ assert '\u212a'.lower() == 'k' # 'âª'
+ self.assertTrue(re.match(r'[19K]', '\u212a', re.I))
+ self.assertTrue(re.match(r'[19k]', '\u212a', re.I))
+ self.assertTrue(re.match(r'[19\u212a]', 'K', re.I))
+ self.assertTrue(re.match(r'[19\u212a]', 'k', re.I))
+ assert '\u017f'.upper() == 'S' # 'Å¿'
+ self.assertTrue(re.match(r'[19S]', '\u017f', re.I))
+ self.assertTrue(re.match(r'[19s]', '\u017f', re.I))
+ self.assertTrue(re.match(r'[19\u017f]', 'S', re.I))
+ self.assertTrue(re.match(r'[19\u017f]', 's', re.I))
+ assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ï¬
', 'ï¬'
+ self.assertTrue(re.match(r'[19\ufb05]', '\ufb06', re.I))
+ self.assertTrue(re.match(r'[19\ufb06]', '\ufb05', re.I))
+
def test_ignore_case_range(self):
# Issues #3511, #17381.
self.assertTrue(re.match(r'[9-a]', '_', re.I))
@@ -620,6 +657,20 @@ class ReTests(unittest.TestCase):
self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428',
re.I))
self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400',
re.I))
+ assert '\u212a'.lower() == 'k' # 'âª'
+ self.assertTrue(re.match(r'[J-M]', '\u212a', re.I))
+ self.assertTrue(re.match(r'[j-m]', '\u212a', re.I))
+ self.assertTrue(re.match(r'[\u2129-\u212b]', 'K', re.I))
+ self.assertTrue(re.match(r'[\u2129-\u212b]', 'k', re.I))
+ assert '\u017f'.upper() == 'S' # 'Å¿'
+ self.assertTrue(re.match(r'[R-T]', '\u017f', re.I))
+ self.assertTrue(re.match(r'[r-t]', '\u017f', re.I))
+ self.assertTrue(re.match(r'[\u017e-\u0180]', 'S', re.I))
+ self.assertTrue(re.match(r'[\u017e-\u0180]', 's', re.I))
+ assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ï¬
', 'ï¬'
+ self.assertTrue(re.match(r'[\ufb04-\ufb05]', '\ufb06', re.I))
+ self.assertTrue(re.match(r'[\ufb06-\ufb07]', '\ufb05', re.I))
+
def test_category(self):
self.assertEqual(re.match(r"(\s)", " ").group(1), " ")
import re
import sys
import unicodedata
from _sre import getlower
from collections import defaultdict
def re_lower(i):
return getlower(i, re.U)
def re_clower(i):
return getlower(ord(i), re.U)
def uname(i):
return unicodedata.name(chr(i), r'U+%04X' % i)
class hexint(int):
def __repr__(self):
return hex(self)
def alpha(i):
c = chr(i)
return c if c.isalpha() else ascii(c)[1:-1]
chars = ''.join(map(chr, range(sys.maxunicode + 1)))
#chars = ''.join(map(chr, range(0x10000)))
equivalences = defaultdict(str)
for c in chars:
equivalences[c.upper()] += c
equivalences = {frozenset(map(re_clower, t))
for t in equivalences.values() if len(t) > 1}
equivalences = sorted(tuple(sorted(t))
for t in equivalences if len(t) > 1)
print('equivalences = (')
for t in equivalences:
print(' # %s' % ', '.join(map(uname, t)))
print(' %r, # %s' % (tuple(map(hexint, t)), ''.join(map(alpha, t))))
print(')')
mapping = {i: tuple(j for j in t if i != j) for t in equivalences for i in t}
print('mapping = {')
for i, t in sorted(mapping.items()):
print(' # %s: %s' % (uname(i), ', '.join(map(uname, t))))
#print(' %s: %r,' % (hex(i), tuple(map(hexint, t))))
print(" %s: %r, # '%s': '%s'" % (hex(i), tuple(map(hexint, t)),
alpha(i), ''.join(map(alpha, t))))
print('}')
_______________________________________________
Python-bugs-list mailing list
Unsubscribe:
https://mail.python.org/mailman/options/python-bugs-list/archive%40mail-archive.com
