https://github.com/python/cpython/commit/52509cc94b1a18cb325dbfa7e5f830b32759a903
commit: 52509cc94b1a18cb325dbfa7e5f830b32759a903
branch: main
author: Loïc Simon <loic.p...@gmail.com>
committer: pablogsal <pablog...@gmail.com>
date: 2025-05-25T17:23:38+01:00
summary:
gh-134582: Fix t-strings untokenize() roundtrip removing space between braces
(#134603)
files:
A Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst
M Lib/test/test_tokenize.py
M Lib/tokenize.py
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 2d41a5e5ac0697..e6b19fe1812d44 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1975,6 +1975,10 @@ def test_roundtrip(self):
for case in cases:
self.check_roundtrip(case)
+ self.check_roundtrip(r"t'{ {}}'")
+ self.check_roundtrip(r"t'{f'{ {}}'}{ {}}'")
+ self.check_roundtrip(r"f'{t'{ {}}'}{ {}}'")
+
def test_continuation(self):
# Balancing continuation
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 8d01fd7bce41b0..559a7aecbde2d1 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -274,7 +274,7 @@ def compat(self, token, iterable):
toks_append = self.tokens.append
startline = token[0] in (NEWLINE, NL)
prevstring = False
- in_fstring = 0
+ in_fstring_or_tstring = 0
for tok in _itertools.chain([token], iterable):
toknum, tokval = tok[:2]
@@ -293,10 +293,10 @@ def compat(self, token, iterable):
else:
prevstring = False
- if toknum == FSTRING_START:
- in_fstring += 1
- elif toknum == FSTRING_END:
- in_fstring -= 1
+ if toknum in {FSTRING_START, TSTRING_START}:
+ in_fstring_or_tstring += 1
+ elif toknum in {FSTRING_END, TSTRING_END}:
+ in_fstring_or_tstring -= 1
if toknum == INDENT:
indents.append(tokval)
continue
@@ -311,8 +311,8 @@ def compat(self, token, iterable):
elif toknum in {FSTRING_MIDDLE, TSTRING_MIDDLE}:
tokval = self.escape_brackets(tokval)
- # Insert a space between two consecutive brackets if we are in an
f-string
- if tokval in {"{", "}"} and self.tokens and self.tokens[-1] ==
tokval and in_fstring:
+ # Insert a space between two consecutive brackets if we are in an
f-string or t-string
+ if tokval in {"{", "}"} and self.tokens and self.tokens[-1] ==
tokval and in_fstring_or_tstring:
tokval = ' ' + tokval
# Insert a space between two consecutive f-strings
diff --git
a/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst
b/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst
new file mode 100644
index 00000000000000..23e1d5891b685f
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst
@@ -0,0 +1 @@
+Fix tokenize.untokenize() round-trip errors related to t-strings braces
escaping
_______________________________________________
Python-checkins mailing list -- python-checkins@python.org
To unsubscribe send an email to python-checkins-le...@python.org
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: arch...@mail-archive.com
