https://github.com/python/cpython/commit/bc5ec7e43267c90e41a1fd03755ae3553b760251 commit: bc5ec7e43267c90e41a1fd03755ae3553b760251 branch: 3.13 author: Pablo Galindo Salgado <[email protected]> committer: pablogsal <[email protected]> date: 2026-09-25T19:20:09Z summary:
[3.13] gh-151464: exclude '<>' token from tokenize output (GH-154854) (#158191) * gh-151464: exclude '<>' token from tokenize output Was: ``` $ echo '1 <> 2' | python -m tokenize 1,0-1,1: NUMBER '1' 1,2-1,4: OP '<>' 1,5-1,6: NUMBER '2' 1,6-1,7: NEWLINE '\n' 2,0-2,0: ENDMARKER '' ``` Now (regardless on ``__future__.barry_as_FLUFL`` import): ``` $ echo '1 <> 2' | ./python -m tokenize 1,0-1,1: NUMBER '1' 1,2-1,3: OP '<' 1,3-1,4: OP '>' 1,5-1,6: NUMBER '2' 1,6-1,7: NEWLINE '\n' 2,0-2,0: ENDMARKER '' ``` in accordance with the Grammar: https://docs.python.org/3.14/reference/lexical_analysis.html#operators-and-delimiters Also adds a custom error message for ``<>`` ("not equal" in Pascal and Python 2). * +1 * address review: lowercase and move invalid rule * address review: news * address review: move test_guido_as_bdfl_ineq_tokens() * address review: revert _PyTokenizer_From* changes * + revert unrelated change * address review: remove whatsnew entry (cherry picked from commit 198bc7683641f6b44d23fe5dc228f732a7cf3df8) Co-authored-by: Sergey B Kirpichev <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst M Grammar/python.gram M Lib/test/test_syntax.py M Lib/test/test_tokenize.py M Parser/action_helpers.c M Parser/lexer/lexer.c M Parser/lexer/state.h M Parser/parser.c M Parser/pegen.c M Parser/pegen.h diff --git a/Grammar/python.gram b/Grammar/python.gram index cc2b58b0fcf49f..c7ece57a889c6a 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -748,6 +748,7 @@ compare_op_bitwise_or_pair[CmpopExprPair*]: | eq_bitwise_or | noteq_bitwise_or | lte_bitwise_or + | invalid_noteq | lt_bitwise_or | gte_bitwise_or | gt_bitwise_or @@ -1439,3 +1440,10 @@ invalid_type_params: RAISE_SYNTAX_ERROR_STARTING_FROM( token, "Type parameter list cannot be empty")} + +invalid_noteq: + | a='<' b='>' { + _PyPegen_tokens_are_adjacent(a, b) + ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '!=' instead of '<>'?") + : NULL + } diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index a7c043a055e59d..fa76d2b5d23c0b 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2728,6 +2728,30 @@ def test_deep_invalid_rule(self): with self.assertRaises(SyntaxError): compile(source, "<string>", "exec") + def test_diamond_operator(self): + self._check_error( + "1<>2", + r"Maybe you meant '!=' instead of '<>'\?", + lineno=1, + end_lineno=1, + offset=2, + end_offset=4, + ) + + def test_diamond_operator_barry_as_flufl(self): + compile( + "from __future__ import barry_as_FLUFL\n1<>2", + "<test>", "exec", + ) + self._check_error( + "from __future__ import barry_as_FLUFL\na != b", + "with Barry as BDFL, use '<>' instead of '!='", + lineno=2, + end_lineno=2, + offset=3, + end_offset=5, + ) + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite()) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 4a7b02ee05b895..606da8fc1f49b9 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1229,6 +1229,20 @@ def test_multiline_non_ascii_fstring_with_expr(self): FSTRING_END \'"\' (2, 2) (2, 3) """) + def test_ineq_tokens(self): + self.check_tokenize("1 != 2", """\ + NUMBER '1' (1, 0) (1, 1) + OP '!=' (1, 2) (1, 4) + NUMBER '2' (1, 5) (1, 6) +""") + self.check_tokenize("1 <> 2", """\ + NUMBER '1' (1, 0) (1, 1) + OP '<' (1, 2) (1, 3) + OP '>' (1, 3) (1, 4) + NUMBER '2' (1, 5) (1, 6) +""") + + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst new file mode 100644 index 00000000000000..ba7dbe13254037 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst @@ -0,0 +1,2 @@ +Exclude invalid token ``<>`` from :mod:`tokenize` output. :exc:`SyntaxError` +for ``<>`` now suggests ``!=``. diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 8deb6b5a303555..6e5e69671af7ac 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1726,6 +1726,7 @@ _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq * na alias_ty alias = asdl_seq_GET(names, i); if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) { p->flags |= PyPARSE_BARRY_AS_BDFL; + p->tok->barry_as_bdfl = 1; } } } diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index e528f2252dd57d..38d3742f6e73d0 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -1144,6 +1144,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t { int c2 = tok_nextc(tok); int current_token = _PyToken_TwoChars(c, c2); + if (c == '<' && c2 == '>' && !tok->barry_as_bdfl) { + current_token = OP; + } if (current_token != OP) { int c3 = tok_nextc(tok); int current_token3 = _PyToken_ThreeChars(c, c2, c3); diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 7e246ee4d3cd6d..f53107c6c26826 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -131,6 +131,7 @@ struct tok_state { #ifdef Py_DEBUG int debug; #endif + int barry_as_bdfl; }; int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, diff --git a/Parser/parser.c b/Parser/parser.c index c91f85fcd5cefc..a4faf0d89440c6 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -333,287 +333,288 @@ static char *soft_keywords[] = { #define invalid_arithmetic_type 1246 #define invalid_factor_type 1247 #define invalid_type_params_type 1248 -#define _loop0_1_type 1249 -#define _loop0_2_type 1250 -#define _loop1_3_type 1251 -#define _loop0_5_type 1252 -#define _gather_4_type 1253 -#define _tmp_6_type 1254 -#define _tmp_7_type 1255 -#define _tmp_8_type 1256 -#define _tmp_9_type 1257 -#define _tmp_10_type 1258 -#define _tmp_11_type 1259 -#define _tmp_12_type 1260 -#define _tmp_13_type 1261 -#define _loop1_14_type 1262 -#define _tmp_15_type 1263 -#define _tmp_16_type 1264 -#define _tmp_17_type 1265 -#define _loop0_19_type 1266 -#define _gather_18_type 1267 -#define _loop0_21_type 1268 -#define _gather_20_type 1269 -#define _tmp_22_type 1270 -#define _tmp_23_type 1271 -#define _loop0_24_type 1272 -#define _loop1_25_type 1273 -#define _loop0_27_type 1274 -#define _gather_26_type 1275 -#define _tmp_28_type 1276 -#define _loop0_30_type 1277 -#define _gather_29_type 1278 -#define _tmp_31_type 1279 -#define _loop1_32_type 1280 -#define _tmp_33_type 1281 -#define _tmp_34_type 1282 -#define _tmp_35_type 1283 -#define _loop0_36_type 1284 -#define _loop0_37_type 1285 -#define _loop0_38_type 1286 -#define _loop1_39_type 1287 -#define _loop0_40_type 1288 -#define _loop1_41_type 1289 -#define _loop1_42_type 1290 -#define _loop1_43_type 1291 -#define _loop0_44_type 1292 -#define _loop1_45_type 1293 -#define _loop0_46_type 1294 -#define _loop1_47_type 1295 -#define _loop0_48_type 1296 -#define _loop0_49_type 1297 -#define _loop1_50_type 1298 -#define _loop0_52_type 1299 -#define _gather_51_type 1300 -#define _loop0_54_type 1301 -#define _gather_53_type 1302 -#define _loop0_56_type 1303 -#define _gather_55_type 1304 -#define _loop0_58_type 1305 -#define _gather_57_type 1306 -#define _tmp_59_type 1307 -#define _loop1_60_type 1308 -#define _loop1_61_type 1309 -#define _tmp_62_type 1310 -#define _tmp_63_type 1311 -#define _loop1_64_type 1312 -#define _loop0_66_type 1313 -#define _gather_65_type 1314 -#define _tmp_67_type 1315 -#define _tmp_68_type 1316 -#define _tmp_69_type 1317 -#define _tmp_70_type 1318 -#define _loop0_72_type 1319 -#define _gather_71_type 1320 -#define _loop0_74_type 1321 -#define _gather_73_type 1322 -#define _tmp_75_type 1323 -#define _loop0_77_type 1324 -#define _gather_76_type 1325 -#define _loop0_79_type 1326 -#define _gather_78_type 1327 -#define _loop0_81_type 1328 -#define _gather_80_type 1329 -#define _loop1_82_type 1330 -#define _loop1_83_type 1331 -#define _loop0_85_type 1332 -#define _gather_84_type 1333 -#define _loop1_86_type 1334 -#define _loop1_87_type 1335 -#define _loop1_88_type 1336 -#define _tmp_89_type 1337 -#define _loop0_91_type 1338 -#define _gather_90_type 1339 -#define _tmp_92_type 1340 -#define _tmp_93_type 1341 -#define _tmp_94_type 1342 -#define _tmp_95_type 1343 -#define _tmp_96_type 1344 -#define _tmp_97_type 1345 -#define _loop0_98_type 1346 -#define _loop0_99_type 1347 -#define _loop0_100_type 1348 -#define _loop1_101_type 1349 -#define _loop0_102_type 1350 -#define _loop1_103_type 1351 -#define _loop1_104_type 1352 -#define _loop1_105_type 1353 -#define _loop0_106_type 1354 -#define _loop1_107_type 1355 -#define _loop0_108_type 1356 -#define _loop1_109_type 1357 -#define _loop0_110_type 1358 -#define _loop1_111_type 1359 -#define _loop0_112_type 1360 -#define _loop0_113_type 1361 -#define _loop1_114_type 1362 -#define _tmp_115_type 1363 -#define _loop0_117_type 1364 -#define _gather_116_type 1365 -#define _loop1_118_type 1366 -#define _loop0_119_type 1367 -#define _loop0_120_type 1368 -#define _tmp_121_type 1369 -#define _tmp_122_type 1370 -#define _loop0_124_type 1371 -#define _gather_123_type 1372 -#define _tmp_125_type 1373 -#define _loop0_127_type 1374 -#define _gather_126_type 1375 -#define _loop0_129_type 1376 -#define _gather_128_type 1377 -#define _loop0_131_type 1378 -#define _gather_130_type 1379 -#define _loop0_133_type 1380 -#define _gather_132_type 1381 -#define _loop0_134_type 1382 -#define _loop0_136_type 1383 -#define _gather_135_type 1384 -#define _loop1_137_type 1385 -#define _tmp_138_type 1386 -#define _loop0_140_type 1387 -#define _gather_139_type 1388 -#define _loop0_142_type 1389 -#define _gather_141_type 1390 -#define _loop0_144_type 1391 -#define _gather_143_type 1392 -#define _loop0_146_type 1393 -#define _gather_145_type 1394 -#define _loop0_148_type 1395 -#define _gather_147_type 1396 -#define _tmp_149_type 1397 -#define _tmp_150_type 1398 -#define _loop0_152_type 1399 -#define _gather_151_type 1400 -#define _tmp_153_type 1401 -#define _tmp_154_type 1402 -#define _tmp_155_type 1403 -#define _tmp_156_type 1404 -#define _tmp_157_type 1405 -#define _tmp_158_type 1406 -#define _tmp_159_type 1407 -#define _tmp_160_type 1408 -#define _tmp_161_type 1409 -#define _tmp_162_type 1410 -#define _loop0_163_type 1411 -#define _loop0_164_type 1412 -#define _loop0_165_type 1413 -#define _tmp_166_type 1414 -#define _tmp_167_type 1415 -#define _tmp_168_type 1416 -#define _tmp_169_type 1417 -#define _loop0_170_type 1418 -#define _loop0_171_type 1419 -#define _loop0_172_type 1420 -#define _loop1_173_type 1421 -#define _tmp_174_type 1422 -#define _loop0_175_type 1423 -#define _tmp_176_type 1424 -#define _loop0_177_type 1425 -#define _loop1_178_type 1426 -#define _tmp_179_type 1427 -#define _tmp_180_type 1428 -#define _tmp_181_type 1429 -#define _loop0_182_type 1430 -#define _tmp_183_type 1431 -#define _tmp_184_type 1432 -#define _loop1_185_type 1433 -#define _tmp_186_type 1434 -#define _loop0_187_type 1435 -#define _loop0_188_type 1436 -#define _loop0_189_type 1437 -#define _loop0_191_type 1438 -#define _gather_190_type 1439 -#define _tmp_192_type 1440 -#define _loop0_193_type 1441 -#define _tmp_194_type 1442 -#define _loop0_195_type 1443 -#define _loop1_196_type 1444 -#define _loop1_197_type 1445 -#define _tmp_198_type 1446 -#define _tmp_199_type 1447 -#define _loop0_200_type 1448 -#define _tmp_201_type 1449 -#define _tmp_202_type 1450 -#define _tmp_203_type 1451 -#define _loop0_205_type 1452 -#define _gather_204_type 1453 -#define _loop0_207_type 1454 -#define _gather_206_type 1455 -#define _loop0_209_type 1456 -#define _gather_208_type 1457 -#define _loop0_211_type 1458 -#define _gather_210_type 1459 -#define _loop0_213_type 1460 -#define _gather_212_type 1461 -#define _tmp_214_type 1462 -#define _loop0_215_type 1463 -#define _loop1_216_type 1464 -#define _tmp_217_type 1465 -#define _loop0_218_type 1466 -#define _loop1_219_type 1467 -#define _tmp_220_type 1468 -#define _tmp_221_type 1469 -#define _tmp_222_type 1470 -#define _tmp_223_type 1471 -#define _tmp_224_type 1472 -#define _tmp_225_type 1473 -#define _tmp_226_type 1474 -#define _tmp_227_type 1475 -#define _tmp_228_type 1476 -#define _tmp_229_type 1477 -#define _tmp_230_type 1478 -#define _loop0_232_type 1479 -#define _gather_231_type 1480 -#define _tmp_233_type 1481 -#define _tmp_234_type 1482 -#define _tmp_235_type 1483 -#define _tmp_236_type 1484 -#define _tmp_237_type 1485 -#define _tmp_238_type 1486 -#define _tmp_239_type 1487 -#define _loop0_240_type 1488 -#define _tmp_241_type 1489 -#define _tmp_242_type 1490 -#define _tmp_243_type 1491 -#define _tmp_244_type 1492 -#define _tmp_245_type 1493 -#define _tmp_246_type 1494 -#define _tmp_247_type 1495 -#define _tmp_248_type 1496 -#define _tmp_249_type 1497 -#define _tmp_250_type 1498 -#define _tmp_251_type 1499 -#define _tmp_252_type 1500 -#define _tmp_253_type 1501 -#define _tmp_254_type 1502 -#define _tmp_255_type 1503 -#define _tmp_256_type 1504 -#define _loop0_257_type 1505 -#define _tmp_258_type 1506 -#define _tmp_259_type 1507 -#define _tmp_260_type 1508 -#define _tmp_261_type 1509 -#define _tmp_262_type 1510 -#define _tmp_263_type 1511 -#define _tmp_264_type 1512 -#define _tmp_265_type 1513 -#define _tmp_266_type 1514 -#define _tmp_267_type 1515 -#define _tmp_268_type 1516 -#define _tmp_269_type 1517 -#define _tmp_270_type 1518 -#define _tmp_271_type 1519 -#define _tmp_272_type 1520 -#define _tmp_273_type 1521 -#define _loop0_275_type 1522 -#define _gather_274_type 1523 -#define _tmp_276_type 1524 -#define _tmp_277_type 1525 -#define _tmp_278_type 1526 -#define _tmp_279_type 1527 -#define _tmp_280_type 1528 -#define _tmp_281_type 1529 +#define invalid_noteq_type 1249 +#define _loop0_1_type 1250 +#define _loop0_2_type 1251 +#define _loop1_3_type 1252 +#define _loop0_5_type 1253 +#define _gather_4_type 1254 +#define _tmp_6_type 1255 +#define _tmp_7_type 1256 +#define _tmp_8_type 1257 +#define _tmp_9_type 1258 +#define _tmp_10_type 1259 +#define _tmp_11_type 1260 +#define _tmp_12_type 1261 +#define _tmp_13_type 1262 +#define _loop1_14_type 1263 +#define _tmp_15_type 1264 +#define _tmp_16_type 1265 +#define _tmp_17_type 1266 +#define _loop0_19_type 1267 +#define _gather_18_type 1268 +#define _loop0_21_type 1269 +#define _gather_20_type 1270 +#define _tmp_22_type 1271 +#define _tmp_23_type 1272 +#define _loop0_24_type 1273 +#define _loop1_25_type 1274 +#define _loop0_27_type 1275 +#define _gather_26_type 1276 +#define _tmp_28_type 1277 +#define _loop0_30_type 1278 +#define _gather_29_type 1279 +#define _tmp_31_type 1280 +#define _loop1_32_type 1281 +#define _tmp_33_type 1282 +#define _tmp_34_type 1283 +#define _tmp_35_type 1284 +#define _loop0_36_type 1285 +#define _loop0_37_type 1286 +#define _loop0_38_type 1287 +#define _loop1_39_type 1288 +#define _loop0_40_type 1289 +#define _loop1_41_type 1290 +#define _loop1_42_type 1291 +#define _loop1_43_type 1292 +#define _loop0_44_type 1293 +#define _loop1_45_type 1294 +#define _loop0_46_type 1295 +#define _loop1_47_type 1296 +#define _loop0_48_type 1297 +#define _loop0_49_type 1298 +#define _loop1_50_type 1299 +#define _loop0_52_type 1300 +#define _gather_51_type 1301 +#define _loop0_54_type 1302 +#define _gather_53_type 1303 +#define _loop0_56_type 1304 +#define _gather_55_type 1305 +#define _loop0_58_type 1306 +#define _gather_57_type 1307 +#define _tmp_59_type 1308 +#define _loop1_60_type 1309 +#define _loop1_61_type 1310 +#define _tmp_62_type 1311 +#define _tmp_63_type 1312 +#define _loop1_64_type 1313 +#define _loop0_66_type 1314 +#define _gather_65_type 1315 +#define _tmp_67_type 1316 +#define _tmp_68_type 1317 +#define _tmp_69_type 1318 +#define _tmp_70_type 1319 +#define _loop0_72_type 1320 +#define _gather_71_type 1321 +#define _loop0_74_type 1322 +#define _gather_73_type 1323 +#define _tmp_75_type 1324 +#define _loop0_77_type 1325 +#define _gather_76_type 1326 +#define _loop0_79_type 1327 +#define _gather_78_type 1328 +#define _loop0_81_type 1329 +#define _gather_80_type 1330 +#define _loop1_82_type 1331 +#define _loop1_83_type 1332 +#define _loop0_85_type 1333 +#define _gather_84_type 1334 +#define _loop1_86_type 1335 +#define _loop1_87_type 1336 +#define _loop1_88_type 1337 +#define _tmp_89_type 1338 +#define _loop0_91_type 1339 +#define _gather_90_type 1340 +#define _tmp_92_type 1341 +#define _tmp_93_type 1342 +#define _tmp_94_type 1343 +#define _tmp_95_type 1344 +#define _tmp_96_type 1345 +#define _tmp_97_type 1346 +#define _loop0_98_type 1347 +#define _loop0_99_type 1348 +#define _loop0_100_type 1349 +#define _loop1_101_type 1350 +#define _loop0_102_type 1351 +#define _loop1_103_type 1352 +#define _loop1_104_type 1353 +#define _loop1_105_type 1354 +#define _loop0_106_type 1355 +#define _loop1_107_type 1356 +#define _loop0_108_type 1357 +#define _loop1_109_type 1358 +#define _loop0_110_type 1359 +#define _loop1_111_type 1360 +#define _loop0_112_type 1361 +#define _loop0_113_type 1362 +#define _loop1_114_type 1363 +#define _tmp_115_type 1364 +#define _loop0_117_type 1365 +#define _gather_116_type 1366 +#define _loop1_118_type 1367 +#define _loop0_119_type 1368 +#define _loop0_120_type 1369 +#define _tmp_121_type 1370 +#define _tmp_122_type 1371 +#define _loop0_124_type 1372 +#define _gather_123_type 1373 +#define _tmp_125_type 1374 +#define _loop0_127_type 1375 +#define _gather_126_type 1376 +#define _loop0_129_type 1377 +#define _gather_128_type 1378 +#define _loop0_131_type 1379 +#define _gather_130_type 1380 +#define _loop0_133_type 1381 +#define _gather_132_type 1382 +#define _loop0_134_type 1383 +#define _loop0_136_type 1384 +#define _gather_135_type 1385 +#define _loop1_137_type 1386 +#define _tmp_138_type 1387 +#define _loop0_140_type 1388 +#define _gather_139_type 1389 +#define _loop0_142_type 1390 +#define _gather_141_type 1391 +#define _loop0_144_type 1392 +#define _gather_143_type 1393 +#define _loop0_146_type 1394 +#define _gather_145_type 1395 +#define _loop0_148_type 1396 +#define _gather_147_type 1397 +#define _tmp_149_type 1398 +#define _tmp_150_type 1399 +#define _loop0_152_type 1400 +#define _gather_151_type 1401 +#define _tmp_153_type 1402 +#define _tmp_154_type 1403 +#define _tmp_155_type 1404 +#define _tmp_156_type 1405 +#define _tmp_157_type 1406 +#define _tmp_158_type 1407 +#define _tmp_159_type 1408 +#define _tmp_160_type 1409 +#define _tmp_161_type 1410 +#define _tmp_162_type 1411 +#define _loop0_163_type 1412 +#define _loop0_164_type 1413 +#define _loop0_165_type 1414 +#define _tmp_166_type 1415 +#define _tmp_167_type 1416 +#define _tmp_168_type 1417 +#define _tmp_169_type 1418 +#define _loop0_170_type 1419 +#define _loop0_171_type 1420 +#define _loop0_172_type 1421 +#define _loop1_173_type 1422 +#define _tmp_174_type 1423 +#define _loop0_175_type 1424 +#define _tmp_176_type 1425 +#define _loop0_177_type 1426 +#define _loop1_178_type 1427 +#define _tmp_179_type 1428 +#define _tmp_180_type 1429 +#define _tmp_181_type 1430 +#define _loop0_182_type 1431 +#define _tmp_183_type 1432 +#define _tmp_184_type 1433 +#define _loop1_185_type 1434 +#define _tmp_186_type 1435 +#define _loop0_187_type 1436 +#define _loop0_188_type 1437 +#define _loop0_189_type 1438 +#define _loop0_191_type 1439 +#define _gather_190_type 1440 +#define _tmp_192_type 1441 +#define _loop0_193_type 1442 +#define _tmp_194_type 1443 +#define _loop0_195_type 1444 +#define _loop1_196_type 1445 +#define _loop1_197_type 1446 +#define _tmp_198_type 1447 +#define _tmp_199_type 1448 +#define _loop0_200_type 1449 +#define _tmp_201_type 1450 +#define _tmp_202_type 1451 +#define _tmp_203_type 1452 +#define _loop0_205_type 1453 +#define _gather_204_type 1454 +#define _loop0_207_type 1455 +#define _gather_206_type 1456 +#define _loop0_209_type 1457 +#define _gather_208_type 1458 +#define _loop0_211_type 1459 +#define _gather_210_type 1460 +#define _loop0_213_type 1461 +#define _gather_212_type 1462 +#define _tmp_214_type 1463 +#define _loop0_215_type 1464 +#define _loop1_216_type 1465 +#define _tmp_217_type 1466 +#define _loop0_218_type 1467 +#define _loop1_219_type 1468 +#define _tmp_220_type 1469 +#define _tmp_221_type 1470 +#define _tmp_222_type 1471 +#define _tmp_223_type 1472 +#define _tmp_224_type 1473 +#define _tmp_225_type 1474 +#define _tmp_226_type 1475 +#define _tmp_227_type 1476 +#define _tmp_228_type 1477 +#define _tmp_229_type 1478 +#define _tmp_230_type 1479 +#define _loop0_232_type 1480 +#define _gather_231_type 1481 +#define _tmp_233_type 1482 +#define _tmp_234_type 1483 +#define _tmp_235_type 1484 +#define _tmp_236_type 1485 +#define _tmp_237_type 1486 +#define _tmp_238_type 1487 +#define _tmp_239_type 1488 +#define _loop0_240_type 1489 +#define _tmp_241_type 1490 +#define _tmp_242_type 1491 +#define _tmp_243_type 1492 +#define _tmp_244_type 1493 +#define _tmp_245_type 1494 +#define _tmp_246_type 1495 +#define _tmp_247_type 1496 +#define _tmp_248_type 1497 +#define _tmp_249_type 1498 +#define _tmp_250_type 1499 +#define _tmp_251_type 1500 +#define _tmp_252_type 1501 +#define _tmp_253_type 1502 +#define _tmp_254_type 1503 +#define _tmp_255_type 1504 +#define _tmp_256_type 1505 +#define _loop0_257_type 1506 +#define _tmp_258_type 1507 +#define _tmp_259_type 1508 +#define _tmp_260_type 1509 +#define _tmp_261_type 1510 +#define _tmp_262_type 1511 +#define _tmp_263_type 1512 +#define _tmp_264_type 1513 +#define _tmp_265_type 1514 +#define _tmp_266_type 1515 +#define _tmp_267_type 1516 +#define _tmp_268_type 1517 +#define _tmp_269_type 1518 +#define _tmp_270_type 1519 +#define _tmp_271_type 1520 +#define _tmp_272_type 1521 +#define _tmp_273_type 1522 +#define _loop0_275_type 1523 +#define _gather_274_type 1524 +#define _tmp_276_type 1525 +#define _tmp_277_type 1526 +#define _tmp_278_type 1527 +#define _tmp_279_type 1528 +#define _tmp_280_type 1529 +#define _tmp_281_type 1530 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -864,6 +865,7 @@ static void *invalid_conversion_character_rule(Parser *p); static void *invalid_arithmetic_rule(Parser *p); static void *invalid_factor_rule(Parser *p); static void *invalid_type_params_rule(Parser *p); +static void *invalid_noteq_rule(Parser *p); static asdl_seq *_loop0_1_rule(Parser *p); static asdl_seq *_loop0_2_rule(Parser *p); static asdl_seq *_loop1_3_rule(Parser *p); @@ -12241,6 +12243,7 @@ comparison_rule(Parser *p) // | eq_bitwise_or // | noteq_bitwise_or // | lte_bitwise_or +// | invalid_noteq // | lt_bitwise_or // | gte_bitwise_or // | gt_bitwise_or @@ -12317,6 +12320,25 @@ compare_op_bitwise_or_pair_rule(Parser *p) D(fprintf(stderr, "%*c%s compare_op_bitwise_or_pair[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lte_bitwise_or")); } + if (p->call_invalid_rules) { // invalid_noteq + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> compare_op_bitwise_or_pair[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_noteq")); + void *invalid_noteq_var; + if ( + (invalid_noteq_var = invalid_noteq_rule(p)) // invalid_noteq + ) + { + D(fprintf(stderr, "%*c+ compare_op_bitwise_or_pair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_noteq")); + _res = invalid_noteq_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s compare_op_bitwise_or_pair[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_noteq")); + } { // lt_bitwise_or if (p->error_indicator) { p->level--; @@ -25449,6 +25471,52 @@ invalid_type_params_rule(Parser *p) return _res; } +// invalid_noteq: '<' '>' +static void * +invalid_noteq_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '<' '>' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_noteq[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'<' '>'")); + Token * a; + Token * b; + if ( + (a = _PyPegen_expect_token(p, 20)) // token='<' + && + (b = _PyPegen_expect_token(p, 21)) // token='>' + ) + { + D(fprintf(stderr, "%*c+ invalid_noteq[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'<' '>'")); + _res = _PyPegen_tokens_are_adjacent ( a , b ) ? RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "invalid syntax. Maybe you meant '!=' instead of '<>'?" ) : NULL; + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_noteq[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'<' '>'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // _loop0_1: NEWLINE static asdl_seq * _loop0_1_rule(Parser *p) diff --git a/Parser/pegen.c b/Parser/pegen.c index 98256046032a8d..311ac0e1046d05 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -945,6 +945,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena PyCompilerFlags *flags, int *errcode, PyObject **interactive_src, PyArena *arena) { + int parser_flags = compute_parser_flags(flags); struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2); if (tok == NULL) { if (PyErr_Occurred()) { @@ -956,6 +957,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena } return NULL; } + tok->barry_as_bdfl = parser_flags & PyPARSE_BARRY_AS_BDFL; if (!tok->fp || ps1 != NULL || ps2 != NULL || PyUnicode_CompareWithASCIIString(filename_ob, "<stdin>") == 0) { tok->fp_interactive = 1; @@ -966,7 +968,6 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena // From here on we need to clean up even if there's an error mod_ty result = NULL; - int parser_flags = compute_parser_flags(flags); Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION, errcode, arena); if (p == NULL) { @@ -995,6 +996,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen PyCompilerFlags *flags, PyArena *arena) { int exec_input = start_rule == Py_file_input; + int parser_flags = compute_parser_flags(flags); struct tok_state *tok; if (flags != NULL && flags->cf_flags & PyCF_IGNORE_COOKIE) { @@ -1012,13 +1014,13 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen } return NULL; } + tok->barry_as_bdfl = parser_flags & PyPARSE_BARRY_AS_BDFL; // This transfers the ownership to the tokenizer tok->filename = Py_NewRef(filename_ob); // We need to clear up from here on mod_ty result = NULL; - int parser_flags = compute_parser_flags(flags); int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ? flags->cf_feature_version : PY_MINOR_VERSION; Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version, diff --git a/Parser/pegen.h b/Parser/pegen.h index 1acfbae861fec3..4555ec74933763 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -196,6 +196,12 @@ RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype, RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, CURRENT_POS, CURRENT_POS, msg, ##__VA_ARGS__) #define RAISE_SYNTAX_ERROR_INVALID_TARGET(type, e) _RAISE_SYNTAX_ERROR_INVALID_TARGET(p, type, e) +Py_LOCAL_INLINE(int) +_PyPegen_tokens_are_adjacent(Token *a, Token *b) +{ + return (a->end_lineno == b->lineno) && (a->end_col_offset == b->col_offset); +} + Py_LOCAL_INLINE(void *) CHECK_CALL(Parser *p, void *result) { _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
