https://github.com/python/cpython/commit/c42790b34f634051750e5da340d17c7da19e4784 commit: c42790b34f634051750e5da340d17c7da19e4784 branch: main author: Petr Viktorin <[email protected]> committer: encukou <[email protected]> date: 2026-09-09T15:57:53+02:00 summary:
gh-155292: Skip updating unicodedata with mismatched interpreter (GH-157066) mkstringprep uses things like str.lower(), so it generates the wrong result if run in an interpreter with a different Unicode data version than the target. This means that updating the Unicode version is a two-step process: run makeunicodedata.py, then compile, then run mkstringprep.py. The two steps can (and should) be combined when re-running regen-unicodedata to verify that the data is up to date. The GH-155292 fix only considered that case. Change makeunicodedata.py to only run mkstringprep.py when the current interpreter is up to it. Otherwise, show a reminder. As an extra complication, download the input (RFC 3454) in the "first step", since an out-of-date stringprep.py's freshness assertion may prevent downloads. Co-authored-by: Maurycy Pawłowski-Wieroński <[email protected]> files: A Misc/NEWS.d/next/Build/2026-09-07-15-13-19.gh-issue-155292._WNxL7.rst M Tools/unicode/makeunicodedata.py M Tools/unicode/mkstringprep.py diff --git a/Misc/NEWS.d/next/Build/2026-09-07-15-13-19.gh-issue-155292._WNxL7.rst b/Misc/NEWS.d/next/Build/2026-09-07-15-13-19.gh-issue-155292._WNxL7.rst new file mode 100644 index 000000000000000..daa9746938d3b6b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-09-07-15-13-19.gh-issue-155292._WNxL7.rst @@ -0,0 +1,3 @@ +In ``makeunicodedata.py``, the script for updating Unicode data, skip +generating ``stringprep.py`` if the current interpreter's Unicode data +version does not match the target version. diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index ab453686d4dfb1a..edb5775eeb1bb66 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -40,6 +40,9 @@ SCRIPT = os.path.normpath(sys.argv[0]) VERSION = "3.3" +# Local cache location +DATA_DIR = os.path.join('Tools', 'unicode', 'data') + # The Unicode Database # -------------------- # When changing UCD version please update @@ -816,10 +819,31 @@ def makeunicodename(unicode, trace): def makestringprep(): FILE = "Lib/stringprep.py" + RFC_LOCAL = os.path.join(DATA_DIR, "rfc3454.txt") + RFC_URL = "https://www.rfc-editor.org/rfc/rfc3454.txt"; + print("--- Preparing", FILE, "...") + # mkstringprep expects a local copy of RFC 3454. Download it now. + # (stringprep is used for URL handling, and if it's not matched + # with the compiled unicodedata, downloads would fail.) + if not os.path.exists(RFC_LOCAL): + download_data(RFC_LOCAL, RFC_URL) + MKSTRINGPREP = "Tools/unicode/mkstringprep.py" + # mkstringprep needs to be run with a Python version that has "its" + # unicode data, since it uses str.lower() and similar. + import unicodedata + if unicodedata.unidata_version != UNIDATA_VERSION: + print() + print("!! Skipping mkstringprep -- mismatched Unicode version !!") + print() + print("Please compile CPython with the updated Unicode database,") + print("then use that interpreter to run:") + print(f" python {MKSTRINGPREP} > {FILE}") + return + with open(FILE, "w") as f: f.truncate() subprocess.check_call([sys.executable, MKSTRINGPREP], stdout=f) @@ -929,19 +953,15 @@ class Difference(Exception):pass normalization_changes)) -DATA_DIR = os.path.join('Tools', 'unicode', 'data') - def open_data(template, version): local = os.path.join(DATA_DIR, template % ('-'+version,)) if not os.path.exists(local): - import urllib.request if version == '3.2.0': # irregular url structure url = ('https://www.unicode.org/Public/3.2-Update/'+template) % ('-'+version,) else: url = ('https://www.unicode.org/Public/%s/ucd/'+template) % (version, '') - os.makedirs(os.path.dirname(local), exist_ok=True) - urllib.request.urlretrieve(url, filename=local) + download_data(local, url) if local.endswith('.txt'): return open(local, encoding='utf-8') else: @@ -949,6 +969,13 @@ def open_data(template, version): return open(local, 'rb') +def download_data(local, url): + import urllib.request + os.makedirs(os.path.dirname(local), exist_ok=True) + print(f'Downloading {url} to {local}') + urllib.request.urlretrieve(url, filename=local) + + def expand_range(char_range: str) -> Iterator[int]: ''' Parses ranges of code points, as described in UAX #44: diff --git a/Tools/unicode/mkstringprep.py b/Tools/unicode/mkstringprep.py index 9740338fef9d3d4..60052aae3b62983 100644 --- a/Tools/unicode/mkstringprep.py +++ b/Tools/unicode/mkstringprep.py @@ -1,11 +1,7 @@ import re -import os import unicodedata as unicodedata_current from unicodedata import ucd_3_2_0 as unicodedata_320 -FILENAME = "Tools/unicode/data/rfc3454.txt" -URL = "https://www.rfc-editor.org/rfc/rfc3454.txt"; - def gen_category(cats): for i in range(0, 0x110000): if unicodedata_320.category(chr(i)) in cats: @@ -52,15 +48,7 @@ def compact_set(l): ############## Read the tables in the RFC ####################### -try: - data_file = open(FILENAME, encoding='utf-8') -except FileNotFoundError: - import urllib.request - os.makedirs(os.path.dirname(FILENAME), exist_ok=True) - urllib.request.urlretrieve(URL, filename=FILENAME) - data_file = open(FILENAME, encoding='utf-8') - -with data_file: +with open("Tools/unicode/data/rfc3454.txt", encoding='utf-8') as data_file: data = data_file.readlines() tables = [] _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
