This is an automated email from the ASF dual-hosted git repository. jlahoda pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push: new 359bf46 [NETBEANS-2430] Ensuring the lexer can be also restarted inside a line, not only at line boundary. (#1222) 359bf46 is described below commit 359bf461d26b7ab5baceb9a3e07bd0c7d36b16a7 Author: Jan Lahoda <jlah...@netbeans.org> AuthorDate: Tue Apr 30 06:48:36 2019 +0200 [NETBEANS-2430] Ensuring the lexer can be also restarted inside a line, not only at line boundary. (#1222) --- .../modules/textmate/lexer/TextmateLexer.java | 58 ++++++++++++++++------ .../modules/textmate/lexer/TextmateLexerTest.java | 57 ++++++++++++++++++--- 2 files changed, 94 insertions(+), 21 deletions(-) diff --git a/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateLexer.java b/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateLexer.java index 168c459..f60a57b 100644 --- a/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateLexer.java +++ b/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateLexer.java @@ -43,32 +43,45 @@ public class TextmateLexer implements Lexer<TextmateTokenId>{ private List<IToken> lineTokens; private int currentIdx; private StackElement state; + private boolean forceReadLine; public TextmateLexer(LexerInput li, Object state, TokenFactory<TextmateTokenId> factory, IGrammar grammar) { - if (state == DO_NOT_RESUME_HERE) { - throw new IllegalStateException("Invalid resume state!"); - } this.li = li; - this.state = (StackElement) state; this.factory = factory; this.grammar = grammar; + if (state instanceof IntralineState) { + IntralineState istate = (IntralineState) state; + this.lineLen = istate.lineLen; + this.currentOffset = istate.currentOffset; + this.lineTokens = istate.lineTokens; + this.currentIdx = istate.currentIdx; + this.state = istate.state; + this.forceReadLine = true; + } else { + this.state = (StackElement) state; + } } @Override public Token<TextmateTokenId> nextToken() { - if (currentOffset >= lineLen) { + if (currentOffset >= lineLen || forceReadLine) { //read next line: int read; while ((read = li.read()) != LexerInput.EOF && read != '\n'); - if (li.readLength() != 0) { - System.err.println(li.readText().toString()); - lineLen = li.readText().length(); - currentOffset = 0; - ITokenizeLineResult tokenized = grammar.tokenizeLine(li.readText().toString(), state); - lineTokens = new ArrayList<>(Arrays.asList(tokenized.getTokens())); - currentIdx = 0; - state = tokenized.getRuleStack(); + if (!forceReadLine) { + if (li.readLength() != 0) { + System.err.println(li.readText().toString()); + lineLen = li.readText().length(); + currentOffset = 0; + ITokenizeLineResult tokenized = grammar.tokenizeLine(li.readText().toString(), state); + lineTokens = new ArrayList<>(Arrays.asList(tokenized.getTokens())); + currentIdx = 0; + state = tokenized.getRuleStack(); + } else { + lineTokens = null; + } } + forceReadLine = false; } if (lineTokens != null && currentIdx < lineTokens.size()) { IToken current = lineTokens.get(currentIdx); @@ -101,10 +114,27 @@ public class TextmateLexer implements Lexer<TextmateTokenId>{ @Override public Object state() { - return lineLen != currentOffset ? DO_NOT_RESUME_HERE : this.state; + return lineLen != currentOffset ? new IntralineState(lineLen, currentOffset, lineTokens, currentIdx, state) : this.state; } @Override public void release() {} + private static final class IntralineState { + private int lineLen; + private int currentOffset; + private List<IToken> lineTokens; + private int currentIdx; + private StackElement state; + + public IntralineState(int lineLen, int currentOffset, List<IToken> lineTokens, int currentIdx, StackElement state) { + this.lineLen = lineLen; + this.currentOffset = currentOffset; + this.lineTokens = lineTokens; + this.currentIdx = currentIdx; + this.state = state; + } + + } + } diff --git a/ide/textmate.lexer/test/unit/src/org/netbeans/modules/textmate/lexer/TextmateLexerTest.java b/ide/textmate.lexer/test/unit/src/org/netbeans/modules/textmate/lexer/TextmateLexerTest.java index a448cfc..2cb2b7c 100644 --- a/ide/textmate.lexer/test/unit/src/org/netbeans/modules/textmate/lexer/TextmateLexerTest.java +++ b/ide/textmate.lexer/test/unit/src/org/netbeans/modules/textmate/lexer/TextmateLexerTest.java @@ -22,9 +22,11 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.util.Arrays; +import javax.swing.text.BadLocationException; import javax.swing.text.Document; import javax.swing.text.PlainDocument; import org.netbeans.api.lexer.Language; +import org.netbeans.api.lexer.Token; import org.netbeans.api.lexer.TokenHierarchy; import org.netbeans.api.lexer.TokenSequence; import org.netbeans.junit.NbTestCase; @@ -116,6 +118,16 @@ public class TextmateLexerTest extends NbTestCase { assertTokenProperties(ts, "test", "string.test"); LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " xbbbx\n"); assertTokenProperties(ts, "test", "string.test"); + LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xcccx xaax \n"); + assertFalse(ts.moveNext()); + doc.remove(3, 1); + ts = hi.tokenSequence(); + LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " "); + assertTokenProperties(ts, "test", "whitespace.test"); + LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xaax"); + assertTokenProperties(ts, "test", "string.test"); + LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " xbbbx\n"); + assertTokenProperties(ts, "test", "string.test"); LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xcccx "); assertTokenProperties(ts, "test", "string.test"); LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xaax"); @@ -126,6 +138,42 @@ public class TextmateLexerTest extends NbTestCase { assertFalse(ts.moveNext()); } + public void testNETBEANS2430() throws Exception { + FileObject grammar = FileUtil.createData(FileUtil.getConfigRoot(), "Editors/text/test/grammar.json"); + try (OutputStream out = grammar.getOutputStream(); + Writer w = new OutputStreamWriter(out)) { + w.write("{ \"scopeName\": \"test\", " + + " \"patterns\": [\n" + + " { \"name\": \"string.test\",\n" + + " \"begin\": \"x([^x]+)x\",\n" + + " \"end\": \"x\\\\1x\"\n" + + " },\n" + + " { \"name\": \"whitespace.test\",\n" + + " \"match\": \" +\"\n" + + " }\n" + + "]}\n"); + } + grammar.setAttribute(LanguageHierarchyImpl.GRAMMAR_MARK, "test"); + Document doc = new PlainDocument(); + doc.insertString(0, " xaax xbbx\nxccx xaax ", null); + doc.putProperty(Language.class, new LanguageHierarchyImpl("text/test", "test").language()); + TokenHierarchy hi = TokenHierarchy.get(doc); + TokenSequence<?> ts = hi.tokenSequence(); + LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " "); + assertTokenProperties(ts, "test", "whitespace.test"); + //do not fully lex + doc.insertString(3, "a", null); + ts = hi.tokenSequence(); + LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " "); + assertTokenProperties(ts, "test", "whitespace.test"); + LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xaaax"); + assertTokenProperties(ts, "test", "string.test"); + LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " xbbx\n"); + assertTokenProperties(ts, "test", "string.test"); + LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xccx xaax \n"); + assertFalse(ts.moveNext()); + } + public void testUTF8() throws Exception { clearWorkDir(); @@ -141,7 +189,7 @@ public class TextmateLexerTest extends NbTestCase { } grammar.setAttribute(LanguageHierarchyImpl.GRAMMAR_MARK, "test"); Document doc = new PlainDocument(); - doc.insertString(0, " večerníček večerníček \n", null); + doc.insertString(0, " večerníček večerníček ", null); doc.putProperty(Language.class, new LanguageHierarchyImpl("text/test", "test").language()); TokenHierarchy<Document> hi = TokenHierarchy.get(doc); TokenSequence<?> ts = hi.tokenSequence(); @@ -155,12 +203,7 @@ public class TextmateLexerTest extends NbTestCase { assertTokenProperties(ts, "test", "ident.test"); LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " "); assertTokenProperties(ts, "test"); - LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " "); - assertTokenProperties(ts, "test", "whitespace.test"); - LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "večerníček"); - assertTokenProperties(ts, "test", "ident.test"); - LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " "); - assertTokenProperties(ts, "test", "whitespace.test"); + LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "\n"); assertFalse(ts.moveNext()); } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@netbeans.apache.org For additional commands, e-mail: commits-h...@netbeans.apache.org For further information about the NetBeans mailing lists, visit: https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists