This is an automated email from the ASF dual-hosted git repository. benweidig pushed a commit to branch property-expression-parser in repository https://gitbox.apache.org/repos/asf/tapestry-5.git
commit 91a073c35c7ccb5552f678f4f517d282403f2c2d Author: Ben Weidig <[email protected]> AuthorDate: Sun Oct 25 12:17:11 2026 +0100 TAP5-2806: Accept every Java identifier character in property expressions The lexer accepted a fixed list of UTF-16 ranges. It rejected Hangul, 'µ', '€' and supplementary characters, and accepted characters that are no Java identifier part, like U+1680. Identifiers are now scanned by code point with Character.isJavaIdentifierStart/Part, and identifier-ignorable characters stay rejected. Keywords still match ASCII letters only, so 'thıs' remains an identifier. The parser imports the lexer's token types, but generateGrammarSource regenerated only the changed grammar. A change to the lexer left the parser with stale token types, and the build cache kept that output. Every grammar change now regenerates both. --- beanmodel/build.gradle | 6 ++ .../internal/antlr/PropertyExpressionLexer.g | 73 +++---------- .../beanmodel/internal/antlr/BaseLexer.java | 120 ++++++++++++++++++++- .../tapestry5/beanmodel/JavaIdentifierTest.java | 27 ++++- 4 files changed, 162 insertions(+), 64 deletions(-) diff --git a/beanmodel/build.gradle b/beanmodel/build.gradle index a4020f246..1ff03b8e9 100644 --- a/beanmodel/build.gradle +++ b/beanmodel/build.gradle @@ -24,6 +24,12 @@ tasks.named('clean') { delete generateGrammarSource.outputDirectory } +tasks.named('generateGrammarSource', AntlrTask) { + // The parser imports the token types of the lexer, but an incremental run regenerates only + // the changed grammar. A non-incremental input makes every change regenerate both. + inputs.files(source).withPropertyName('allGrammars').withPathSensitivity(PathSensitivity.RELATIVE) +} + tasks.named('compileJava', JavaCompile) { dependsOn generateGrammarSource options.fork = true diff --git a/beanmodel/src/main/antlr/org/apache/tapestry5/beanmodel/internal/antlr/PropertyExpressionLexer.g b/beanmodel/src/main/antlr/org/apache/tapestry5/beanmodel/internal/antlr/PropertyExpressionLexer.g index eedf98d47..0d7cea78e 100644 --- a/beanmodel/src/main/antlr/org/apache/tapestry5/beanmodel/internal/antlr/PropertyExpressionLexer.g +++ b/beanmodel/src/main/antlr/org/apache/tapestry5/beanmodel/internal/antlr/PropertyExpressionLexer.g @@ -61,65 +61,20 @@ COLON : ':'; fragment QUOTE : '\''; -// Clumsy but effective approach to case-insensitive identifiers. - -fragment A - : ('a' | 'A'); -fragment E - : ('e' | 'E'); -fragment F - : ('f' | 'F'); -fragment H - : ('h' | 'H'); -fragment I - : ('i' | 'I'); -fragment L - : ('l' | 'L'); -fragment N - : ('n'|'N'); -fragment R - : ('r' | 'R'); -fragment S - : ('s' | 'S'); -fragment T - : ('t' | 'T'); -fragment U - : ('u' | 'U'); - -// Identifiers are case insensitive - -NULL : N U L L; -TRUE : T R U E; -FALSE : F A L S E; -THIS : T H I S; - -IDENTIFIER - : JAVA_ID_START (JAVA_ID_PART)* - ; - -fragment -JAVA_ID_START - : '\u0024' - | '\u0041'..'\u005a' - | '\u005f' - | '\u0061'..'\u007a' - | '\u00c0'..'\u00d6' - | '\u00d8'..'\u00f6' - | '\u00f8'..'\u00ff' - | '\u0100'..'\u1fff' - | '\u3040'..'\u318f' - | '\u3300'..'\u337f' - | '\u3400'..'\u3d2d' - | '\u4e00'..'\u9fff' - | '\uf900'..'\ufaff' - ; - -fragment -JAVA_ID_PART - : JAVA_ID_START - | '\u0030'..'\u0039' - ; - +// Identifiers and the case-insensitive keywords are scanned by BaseLexer.nextToken(), +// because their characters are defined by Character.isJavaIdentifierStart/Part. +// These rules only provide the token types. + +fragment NULL + : {this.getClass(); /* Fix java.lang.VerifyError: Stack size too large */}; +fragment TRUE + : {this.getClass(); /* Fix java.lang.VerifyError: Stack size too large */}; +fragment FALSE + : {this.getClass(); /* Fix java.lang.VerifyError: Stack size too large */}; +fragment THIS + : {this.getClass(); /* Fix java.lang.VerifyError: Stack size too large */}; +fragment IDENTIFIER + : {this.getClass(); /* Fix java.lang.VerifyError: Stack size too large */}; // The Safe Dereference operator understands not to de-reference through // a null. diff --git a/beanmodel/src/main/java/org/apache/tapestry5/beanmodel/internal/antlr/BaseLexer.java b/beanmodel/src/main/java/org/apache/tapestry5/beanmodel/internal/antlr/BaseLexer.java index 576a1bc2a..a8e0d1c0c 100644 --- a/beanmodel/src/main/java/org/apache/tapestry5/beanmodel/internal/antlr/BaseLexer.java +++ b/beanmodel/src/main/java/org/apache/tapestry5/beanmodel/internal/antlr/BaseLexer.java @@ -1,4 +1,4 @@ -// Copyright 2008, 2009 The Apache Software Foundation +// Copyright 2008, 2009, 2026 The Apache Software Foundation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import org.antlr.runtime.CharStream; import org.antlr.runtime.Lexer; import org.antlr.runtime.RecognizerSharedState; import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.Token; public abstract class BaseLexer extends Lexer { @@ -44,6 +45,123 @@ public abstract class BaseLexer extends Lexer } } + /** + * Scans identifiers and keywords here, and leaves every other token to the generated lexer. + * An identifier is a Java identifier without identifier-ignorable characters, read by code point. + * A keyword is an identifier that matches NULL, TRUE, FALSE or THIS, ignoring ASCII case. + */ + @Override + public Token nextToken() + { + // Whitespace has to go first, or the generated lexer would skip it and then read the identifier behind it. + + while (isWhitespace(input.LA(1))) + { + input.consume(); + } + + if (!isIdentifierStart(codePointAt(1))) + { + return super.nextToken(); + } + + state.token = null; + state.channel = Token.DEFAULT_CHANNEL; + state.tokenStartCharIndex = input.index(); + state.tokenStartCharPositionInLine = input.getCharPositionInLine(); + state.tokenStartLine = input.getLine(); + state.text = null; + + do + { + consumeCodePoint(); + } while (isIdentifierPart(codePointAt(1))); + + state.type = identifierType(getText()); + + return emit(); + } + + private static boolean isWhitespace(int c) + { + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; + } + + private static boolean isIdentifierStart(int codePoint) + { + return Character.isJavaIdentifierStart(codePoint) && !Character.isIdentifierIgnorable(codePoint); + } + + private static boolean isIdentifierPart(int codePoint) + { + return Character.isJavaIdentifierPart(codePoint) && !Character.isIdentifierIgnorable(codePoint); + } + + /** + * Returns the code point starting at the given lookahead position, or {@link CharStream#EOF}. + */ + private int codePointAt(int i) + { + int high = input.LA(i); + + if (high != CharStream.EOF && Character.isHighSurrogate((char) high)) + { + int low = input.LA(i + 1); + + if (low != CharStream.EOF && Character.isLowSurrogate((char) low)) + { + return Character.toCodePoint((char) high, (char) low); + } + } + + return high; + } + + private void consumeCodePoint() + { + boolean supplementary = Character.isSupplementaryCodePoint(codePointAt(1)); + + input.consume(); + + if (supplementary) + { + input.consume(); + } + } + + private static int identifierType(String text) + { + // Only ASCII letters fold, so that 'thıs' (dotless i) stays an identifier. + + StringBuilder folded = new StringBuilder(text.length()); + + for (int i = 0; i < text.length(); i++) + { + char c = text.charAt(i); + + if (c >= 'A' && c <= 'Z') + { + c = (char) (c + ('a' - 'A')); + } + + folded.append(c); + } + + switch (folded.toString()) + { + case "null": + return PropertyExpressionLexer.NULL; + case "true": + return PropertyExpressionLexer.TRUE; + case "false": + return PropertyExpressionLexer.FALSE; + case "this": + return PropertyExpressionLexer.THIS; + default: + return PropertyExpressionLexer.IDENTIFIER; + } + } + @Override public void reportError(RecognitionException e) { diff --git a/beanmodel/src/test/java/org/apache/tapestry5/beanmodel/JavaIdentifierTest.java b/beanmodel/src/test/java/org/apache/tapestry5/beanmodel/JavaIdentifierTest.java index 628940f96..c446a5a6d 100644 --- a/beanmodel/src/test/java/org/apache/tapestry5/beanmodel/JavaIdentifierTest.java +++ b/beanmodel/src/test/java/org/apache/tapestry5/beanmodel/JavaIdentifierTest.java @@ -18,7 +18,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.tapestry5.beanmodel.services.PropertyConduitSource; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -49,7 +48,6 @@ public class JavaIdentifierTest assertTrue(ex.getMessage().startsWith(prefix), ex.getMessage()); } - @Disabled("The ANTLR lexer accepts only a fixed list of character ranges") @ParameterizedTest @ValueSource(strings = {"한글", "µx", "€y", "㴮z", "𠀀w"}) void java_identifier_start_characters_are_accepted(String expression) @@ -57,14 +55,12 @@ public class JavaIdentifierTest assertEquals(expression + "-value", get(expression)); } - @Disabled("The ANTLR lexer accepts only a fixed list of character ranges") @Test void hangul_identifier_heads_a_chain() { assertEquals("b-value", get("가.b")); } - @Disabled("The ANTLR lexer accepts U+1680 and U+0378 in identifiers") @ParameterizedTest @ValueSource(strings = {"\u1680x", "\u0378y"}) void characters_that_are_no_java_identifier_start_are_rejected(String expression) @@ -72,6 +68,19 @@ public class JavaIdentifierTest assertParseError(expression); } + @Test + void whitespace_around_non_ascii_identifier_is_ignored() + { + assertEquals("b-value", get(" 가 . b ")); + } + + @ParameterizedTest + @ValueSource(strings = {"thıs", "falſe"}) + void keyword_lookalikes_with_non_ascii_letters_are_identifiers(String expression) + { + assertEquals(expression + "-value", get(expression)); + } + @ParameterizedTest @ValueSource(strings = {"\u0000", "\u0007", "\u007f", "\u00ad", "\u200b", "\u200d", "\ufeff"}) void identifier_ignorable_characters_are_rejected(String ignorable) @@ -115,6 +124,16 @@ public class JavaIdentifierTest { return "foo-value"; } + + public String getThıs() + { + return "thıs-value"; + } + + public String getFalſe() + { + return "falſe-value"; + } } public static class Hangul
