Author: cbrisson
Date: Sat Nov 30 13:47:07 2019
New Revision: 1870626
URL: http://svn.apache.org/viewvc?rev=1870626&view=rev
Log:
[core] Use devoted File Separator control character for parser end of template
processing
Modified:
velocity/engine/trunk/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java
velocity/engine/trunk/velocity-engine-core/src/main/parser/Parser.jjt
velocity/engine/trunk/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java
Modified:
velocity/engine/trunk/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java
URL:
http://svn.apache.org/viewvc/velocity/engine/trunk/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java?rev=1870626&r1=1870625&r2=1870626&view=diff
==============================================================================
---
velocity/engine/trunk/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java
(original)
+++
velocity/engine/trunk/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java
Sat Nov 30 13:47:07 2019
@@ -61,9 +61,11 @@ implements CharStream
private int inBuf = 0;
/* CB - to properly handle EOF *inside* javacc lexer,
- * we send a 'zero-width whitespace' *just before* EOF
+ * we send a 'file separator' ascii char *just before* EOF
+ * (see https://en.wikipedia.org/wiki/Delimiter#ASCII_delimited_text)
*/
private boolean beforeEOF = false;
+ private static char END_OF_FILE = '\u001C';
private void ExpandBuff(boolean wrapAround)
{
@@ -161,7 +163,7 @@ implements CharStream
inputStream.close();
throw new java.io.IOException();
}
- buffer[maxNextCharInd++] = '\u200B';
+ buffer[maxNextCharInd++] = END_OF_FILE;
beforeEOF = true;
}
else
Modified: velocity/engine/trunk/velocity-engine-core/src/main/parser/Parser.jjt
URL:
http://svn.apache.org/viewvc/velocity/engine/trunk/velocity-engine-core/src/main/parser/Parser.jjt?rev=1870626&r1=1870625&r2=1870626&view=diff
==============================================================================
--- velocity/engine/trunk/velocity-engine-core/src/main/parser/Parser.jjt
(original)
+++ velocity/engine/trunk/velocity-engine-core/src/main/parser/Parser.jjt Sat
Nov 30 13:47:07 2019
@@ -749,7 +749,7 @@ TOKEN_MGR_DECLS:
<PRE_DIRECTIVE,PRE_REFERENCE,PRE_OLD_REFERENCE>
TOKEN :
{
- <LONE_SYMBOL: "\u200B" >
+ <LONE_SYMBOL: "\u001C" >
{
stateStackPop();
}
@@ -759,7 +759,7 @@ TOKEN :
<REFERENCE,REFMODIFIER,OLD_REFMODIFIER,REFMOD3,REFINDEX,DIRECTIVE,REFMOD2,DEFAULT,REFMOD,IN_TEXTBLOCK,IN_MULTILINE_COMMENT,IN_FORMAL_COMMENT,IN_SINGLE_LINE_COMMENT>
TOKEN :
{
- <ZERO_WIDTH_WHITESPACE: "\u200B">
+ <ZERO_WIDTH_WHITESPACE: "\u001C">
}
<REFERENCE, REFMODIFIER, OLD_REFMODIFIER, REFMOD3>
@@ -1031,7 +1031,7 @@ MORE :
}
}
-| <"${parser.char.hash}${parser.char.asterisk}${parser.char.asterisk}"
~["${parser.char.hash}","\u200B"]>
+| <"${parser.char.hash}${parser.char.asterisk}${parser.char.asterisk}"
~["${parser.char.hash}","\u001C"]>
{
if (!inComment)
{
@@ -1179,7 +1179,7 @@ SKIP :
<IN_TEXTBLOCK>
MORE :
{
- < ~["\u200B"] >
+ < ~["\u001C"] >
}
/* -----------------------------------------------------------------------
@@ -1219,7 +1219,7 @@ TOKEN :
// <STRING_LITERAL: ( "\"" ( ~["\"","\n","\r"] )* "\"" ) | ( "'" (
~["'","\n","\r"] )* "'" ) >
< STRING_LITERAL:
("\""
- ( (~["\"","\u200B"])
+ ( (~["\"","\u001C"])
| ("\\"
( ["n","t","b","r","f"]
| ["0"-"7"] ( ["0"-"7"] )?
@@ -1234,7 +1234,7 @@ TOKEN :
)
|
("\'"
- ( (~["\'","\u200B"])
+ ( (~["\'","\u001C"])
| ("''")
| ( "\\" (" ")* "\n")
)*
@@ -1526,12 +1526,12 @@ TOKEN :
{
<DOUBLE_ESCAPE : "\\\\">
| <ESCAPE: "\\" >
-| <TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r",
"\n","\u200B"])* (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r",
"\n", " ", "\t","\u200B"])+ (~["${parser.char.dollar}", "${parser.char.hash}",
"\\", "\r", "\n","\u200B"])* <NEWLINE> ((~["${parser.char.dollar}",
"${parser.char.hash}", "\\", "\r", "\n","\u200B"])* <NEWLINE>)* >
+| <TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r",
"\n","\u001C"])* (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r",
"\n", " ", "\t","\u001C"])+ (~["${parser.char.dollar}", "${parser.char.hash}",
"\\", "\r", "\n","\u001C"])* <NEWLINE> ((~["${parser.char.dollar}",
"${parser.char.hash}", "\\", "\r", "\n","\u001C"])* <NEWLINE>)* >
}
TOKEN :
{
- <INLINE_TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\",
"\r", "\n","\u200B"])+ >
+ <INLINE_TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\",
"\r", "\n","\u001C"])+ >
}
/**
Modified:
velocity/engine/trunk/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java
URL:
http://svn.apache.org/viewvc/velocity/engine/trunk/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java?rev=1870626&r1=1870625&r2=1870626&view=diff
==============================================================================
---
velocity/engine/trunk/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java
(original)
+++
velocity/engine/trunk/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java
Sat Nov 30 13:47:07 2019
@@ -28,4 +28,10 @@ public class Velocity919TestCase extends
assertEvalEquals("before\u200Bafter", "before\u200Bafter");
}
+ public void testUserFileSeparator() throws Exception
+ {
+ assertEvalEquals("before\u001Cafter", "before\u001Cafter");
+ }
+
}
+
