This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 8d00daa59c fix highlighting on windows, fixes #7971 (#7988)
8d00daa59c is described below
commit 8d00daa59cb3bd1722ae76d400916b20448a835c
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Mon Aug 17 20:53:34 2026 +0200
fix highlighting on windows, fixes #7971 (#7988)
---
.../hop/ui/hopgui/ContentEditorTm4eSupport.java | 205 +++++++++++----------
.../ui/hopgui/ContentEditorTm4eSupportTest.java | 130 +++++++++++++
2 files changed, 235 insertions(+), 100 deletions(-)
diff --git
a/rcp/src/main/java/org/apache/hop/ui/hopgui/ContentEditorTm4eSupport.java
b/rcp/src/main/java/org/apache/hop/ui/hopgui/ContentEditorTm4eSupport.java
index 928c0765ec..34bcebfe61 100644
--- a/rcp/src/main/java/org/apache/hop/ui/hopgui/ContentEditorTm4eSupport.java
+++ b/rcp/src/main/java/org/apache/hop/ui/hopgui/ContentEditorTm4eSupport.java
@@ -214,8 +214,7 @@ final class ContentEditorTm4eSupport {
new org.eclipse.jface.text.presentation.PresentationReconciler();
reconciler.setDocumentPartitioning(
org.eclipse.jface.text.IDocumentExtension3.DEFAULT_PARTITIONING);
- Tm4eDamagerRepairer repairer =
- new Tm4eDamagerRepairer(grammar, support::scopeToAttribute,
support.display);
+ Tm4eDamagerRepairer repairer = new Tm4eDamagerRepairer(grammar,
support::scopeToAttribute);
reconciler.setDamager(repairer,
org.eclipse.jface.text.IDocument.DEFAULT_CONTENT_TYPE);
reconciler.setRepairer(repairer,
org.eclipse.jface.text.IDocument.DEFAULT_CONTENT_TYPE);
return reconciler;
@@ -356,62 +355,80 @@ final class ContentEditorTm4eSupport {
return dark ? D_DEFAULT : L_DEFAULT;
}
+ /** A stretch of text and the TM4E scopes covering it. No scopes means
unscoped text. */
+ record ScopedRange(int offset, int length, List<String> scopes) {}
+
+ /**
+ * Tokenizes the text with the grammar and returns the part covering
[rangeOffset, rangeOffset +
+ * rangeLength) as consecutive scoped ranges.
+ *
+ * <p>The ranges tile that interval: every character is covered exactly
once, including the line
+ * delimiters TM4E doesn't tokenize. That is not cosmetic. {@link
+ * org.eclipse.jface.text.rules.DefaultDamagerRepairer#createPresentation}
merges neighbouring
+ * tokens that share a {@link TextAttribute} by adding up their lengths,
without looking at their
+ * offsets, so a gap in the stream shortens the resulting style range by the
size of that gap.
+ * Leaving the delimiters out cost a run of same-coloured lines one
character per line with LF and
+ * two with CRLF, which is why highlighting stopped short of the end of a
run and why it looked
+ * like a Windows bug (issue #7971).
+ */
+ static List<ScopedRange> tokenize(
+ IGrammar grammar, String text, int rangeOffset, int rangeLength) {
+ int rangeEnd = Math.min(rangeOffset + rangeLength, text.length());
+ List<ScopedRange> ranges = new java.util.ArrayList<>();
+ int cursor = Math.max(rangeOffset, 0);
+
+ for (ScopedRange token : tokenizeLines(grammar, text, cursor, rangeEnd)) {
+ if (token.offset() > cursor) {
+ // Text no token covers, i.e. a line delimiter: keep the stream
contiguous
+ ranges.add(new ScopedRange(cursor, token.offset() - cursor,
List.of()));
+ }
+ ranges.add(token);
+ cursor = token.offset() + token.length();
+ }
+ if (cursor < rangeEnd) {
+ ranges.add(new ScopedRange(cursor, rangeEnd - cursor, List.of()));
+ }
+ return ranges;
+ }
+
/** Damager/repairer that uses TM4E to tokenize and applies our attributes.
*/
private static final class Tm4eDamagerRepairer
extends org.eclipse.jface.text.rules.DefaultDamagerRepairer {
- private final IGrammar grammar;
- private final java.util.function.Function<List<String>,
org.eclipse.jface.text.TextAttribute>
- scopeToAttr;
- private final Display display;
Tm4eDamagerRepairer(
IGrammar grammar,
- java.util.function.Function<List<String>,
org.eclipse.jface.text.TextAttribute> scopeToAttr,
- Display display) {
- super(new Tm4eScanner(grammar, scopeToAttr, display));
- this.grammar = grammar;
- this.scopeToAttr = scopeToAttr;
- this.display = display;
+ java.util.function.Function<List<String>,
org.eclipse.jface.text.TextAttribute>
+ scopeToAttr) {
+ super(new Tm4eScanner(grammar, scopeToAttr));
}
}
/** JFace ITokenScanner that tokenizes with TM4E and returns tokens with our
attributes. */
private static final class Tm4eScanner implements
org.eclipse.jface.text.rules.ITokenScanner {
- private static final int MAX_LINES_TO_TOKENIZE = 100_000;
- private static final int MAX_LINE_LENGTH = 100_000;
-
private final IGrammar grammar;
private final java.util.function.Function<List<String>,
org.eclipse.jface.text.TextAttribute>
scopeToAttr;
- private final Display display;
- private IDocument document;
- private int rangeOffset;
- private int rangeLength;
- private java.util.List<ColoredToken> tokens;
+ private java.util.List<ScopedRange> ranges;
private int index;
private int tokenOffset;
private int tokenLength;
Tm4eScanner(
IGrammar grammar,
- java.util.function.Function<List<String>,
org.eclipse.jface.text.TextAttribute> scopeToAttr,
- Display display) {
+ java.util.function.Function<List<String>,
org.eclipse.jface.text.TextAttribute>
+ scopeToAttr) {
this.grammar = grammar;
this.scopeToAttr = scopeToAttr;
- this.display = display;
}
@Override
public void setRange(IDocument doc, int offset, int length) {
- this.document = doc;
- this.rangeOffset = offset;
- this.rangeLength = length;
try {
- this.tokens = tokenize(doc, offset, length);
+ this.ranges = tokenize(grammar, doc.get(), offset, length);
} catch (Exception ignored) {
- this.tokens = Collections.emptyList();
+ this.ranges = Collections.emptyList();
}
this.index = 0;
this.tokenOffset = 0;
@@ -420,14 +437,14 @@ final class ContentEditorTm4eSupport {
@Override
public IToken nextToken() {
- if (tokens == null || index >= tokens.size()) {
+ if (ranges == null || index >= ranges.size()) {
tokenOffset = tokenLength = 0;
return Token.EOF;
}
- ColoredToken t = tokens.get(index++);
- tokenOffset = t.offset;
- tokenLength = t.length;
- return new Token(t.attribute);
+ ScopedRange range = ranges.get(index++);
+ tokenOffset = range.offset();
+ tokenLength = range.length();
+ return new Token(scopeToAttr.apply(range.scopes()));
}
@Override
@@ -439,84 +456,72 @@ final class ContentEditorTm4eSupport {
public int getTokenLength() {
return tokenLength;
}
+ }
- /** Tokenize document; runs synchronously on the UI thread. */
- private java.util.List<ColoredToken> tokenize(IDocument doc, int
rangeOffset, int rangeLength) {
- String text;
- try {
- text = doc.get();
- } catch (Exception e) {
- return Collections.emptyList();
- }
- return tokenizeLines(text, rangeOffset, rangeLength);
- }
+ private static final int MAX_LINES_TO_TOKENIZE = 100_000;
+ private static final int MAX_LINE_LENGTH = 100_000;
- private java.util.List<ColoredToken> tokenizeLines(
- String text, int rangeOffset, int rangeLength) {
- java.util.List<ColoredToken> result = new java.util.ArrayList<>();
- try {
- int len = text.length();
- String[] lines = text.split("\\n", -1);
- long lineStart = 0;
- IStateStack state = null;
- int rangeEnd = Math.min(rangeOffset + rangeLength, len);
-
- for (int lineIndex = 0; lineIndex < lines.length; lineIndex++) {
- if (lineIndex >= MAX_LINES_TO_TOKENIZE) {
- break;
- }
+ /** The scoped ranges the grammar produces, line by line, clipped to the
requested range. */
+ private static List<ScopedRange> tokenizeLines(
+ IGrammar grammar, String text, int rangeOffset, int rangeEnd) {
+ List<ScopedRange> result = new java.util.ArrayList<>();
+ try {
+ String[] lines = text.split("\\n", -1);
+ long lineStart = 0;
+ IStateStack state = null;
- String raw = lines[lineIndex];
- int rawLen = raw.length();
- String line = raw;
- if (rawLen > 0 && raw.charAt(rawLen - 1) == '\r') {
- line = raw.substring(0, rawLen - 1);
- }
- int lineLen = line.length();
- if (lineLen > MAX_LINE_LENGTH) {
- lineStart += rawLen + 1L;
- continue;
- }
+ for (int lineIndex = 0; lineIndex < lines.length; lineIndex++) {
+ if (lineIndex >= MAX_LINES_TO_TOKENIZE) {
+ break;
+ }
- long lineEnd = lineStart + rawLen;
- if (lineEnd <= rangeOffset) {
- ITokenizeLineResult<org.eclipse.tm4e.core.grammar.IToken[]> res =
- grammar.tokenizeLine(line, state, null);
- state = res.getRuleStack();
- lineStart = lineEnd + 1;
- continue;
- }
- if (lineStart >= rangeEnd) break;
+ String raw = lines[lineIndex];
+ int rawLen = raw.length();
+ String line = raw;
+ if (rawLen > 0 && raw.charAt(rawLen - 1) == '\r') {
+ line = raw.substring(0, rawLen - 1);
+ }
+ if (line.length() > MAX_LINE_LENGTH) {
+ lineStart += rawLen + 1L;
+ continue;
+ }
+ long lineEnd = lineStart + rawLen;
+ if (lineEnd <= rangeOffset) {
ITokenizeLineResult<org.eclipse.tm4e.core.grammar.IToken[]> res =
grammar.tokenizeLine(line, state, null);
state = res.getRuleStack();
-
- for (org.eclipse.tm4e.core.grammar.IToken t : res.getTokens()) {
- long tStartLong = lineStart + t.getStartIndex();
- long tEndLong = lineStart + t.getEndIndex();
- if (tStartLong > Integer.MAX_VALUE || tEndLong >
Integer.MAX_VALUE) continue;
- int tStart = (int) tStartLong;
- int tEnd = (int) tEndLong;
- if (tEnd <= rangeOffset || tStart >= rangeEnd) continue;
- int o = Math.max(tStart, rangeOffset);
- int l = Math.min(tEnd, rangeEnd) - o;
- if (l <= 0) continue;
- List<String> tokenScopes = t.getScopes();
- if (ContentEditorTm4eSupport.TRACE_SCOPES && tokenScopes != null) {
- String joined = String.join(" ", tokenScopes);
- System.err.println("[TM4E token] offset=" + o + " len=" + l + "
| " + joined);
- }
- result.add(new ColoredToken(o, l, scopeToAttr.apply(tokenScopes)));
- }
lineStart = lineEnd + 1;
+ continue;
}
- } catch (Exception ignored) {
- // ignore
+ if (lineStart >= rangeEnd) break;
+
+ ITokenizeLineResult<org.eclipse.tm4e.core.grammar.IToken[]> res =
+ grammar.tokenizeLine(line, state, null);
+ state = res.getRuleStack();
+
+ for (org.eclipse.tm4e.core.grammar.IToken t : res.getTokens()) {
+ long tStartLong = lineStart + t.getStartIndex();
+ long tEndLong = lineStart + t.getEndIndex();
+ if (tStartLong > Integer.MAX_VALUE || tEndLong > Integer.MAX_VALUE)
continue;
+ int tStart = (int) tStartLong;
+ int tEnd = (int) tEndLong;
+ if (tEnd <= rangeOffset || tStart >= rangeEnd) continue;
+ int o = Math.max(tStart, rangeOffset);
+ int l = Math.min(tEnd, rangeEnd) - o;
+ if (l <= 0) continue;
+ List<String> tokenScopes = t.getScopes();
+ if (TRACE_SCOPES && tokenScopes != null) {
+ System.err.println(
+ "[TM4E token] offset=" + o + " len=" + l + " | " +
String.join(" ", tokenScopes));
+ }
+ result.add(new ScopedRange(o, l, tokenScopes == null ? List.of() :
tokenScopes));
+ }
+ lineStart = lineEnd + 1;
}
- return result;
+ } catch (Exception ignored) {
+ // ignore
}
-
- private record ColoredToken(int offset, int length, TextAttribute
attribute) {}
+ return result;
}
}
diff --git
a/rcp/src/test/java/org/apache/hop/ui/hopgui/ContentEditorTm4eSupportTest.java
b/rcp/src/test/java/org/apache/hop/ui/hopgui/ContentEditorTm4eSupportTest.java
index 9a8a8d267b..91fbe1502b 100644
---
a/rcp/src/test/java/org/apache/hop/ui/hopgui/ContentEditorTm4eSupportTest.java
+++
b/rcp/src/test/java/org/apache/hop/ui/hopgui/ContentEditorTm4eSupportTest.java
@@ -161,6 +161,136 @@ class ContentEditorTm4eSupportTest {
assertTrue(result.getTokens().length > 0, "batch line should produce
tokens");
}
+ /**
+ * The scoped ranges have to tile the requested range without holes. JFace
merges neighbouring
+ * tokens that share a TextAttribute by adding up their lengths, so a hole
(a line delimiter that
+ * no token covers) shortens the style range of a run of same-coloured
lines: one character per
+ * line with LF, two with CRLF. That is issue #7971 - highlighting that
stops before the end of a
+ * run, worse on Windows and worse the more lines are involved.
+ */
+ private static void assertTiles(IGrammar grammar, String text) {
+ List<ContentEditorTm4eSupport.ScopedRange> ranges =
+ ContentEditorTm4eSupport.tokenize(grammar, text, 0, text.length());
+
+ int expectedOffset = 0;
+ for (ContentEditorTm4eSupport.ScopedRange range : ranges) {
+ assertEquals(
+ expectedOffset,
+ range.offset(),
+ () ->
+ "gap or overlap before ["
+ + text.substring(range.offset(), range.offset() +
range.length())
+ + "] at offset "
+ + range.offset());
+ assertTrue(range.length() > 0, "empty range at offset " +
range.offset());
+ expectedOffset += range.length();
+ }
+ assertEquals(text.length(), expectedOffset, "ranges stop before the end of
the text");
+ }
+
+ @Test
+ void tokenize_coversEveryCharacter_withUnixLineEndings() throws Exception {
+ IGrammar grammar = loadGrammar("source.sql", "sql.json");
+
+ assertTiles(grammar, "SELECT\n p.CODE\nfrom\n DIM_PRESTATION p\n");
+ }
+
+ @Test
+ void tokenize_coversEveryCharacter_withWindowsLineEndings() throws Exception
{
+ IGrammar grammar = loadGrammar("source.sql", "sql.json");
+
+ assertTiles(grammar, "SELECT\r\n p.CODE\r\nfrom\r\n DIM_PRESTATION
p\r\n");
+ }
+
+ @Test
+ void tokenize_coversEveryCharacter_ofRepeatedHeadings() throws Exception {
+ IGrammar grammar = loadGrammar("text.html.markdown", "markdown.json");
+
+ // Consecutive lines that share a colour are what makes the shortfall add
up
+ assertTiles(grammar, "# h1\n## h2\n### h3\n\n# h1\n## h2\n### h3\n####
h4\n");
+ assertTiles(grammar, "# h1\r\n## h2\r\n### h3\r\n\r\n# h1\r\n## h2\r\n###
h3\r\n#### h4\r\n");
+ }
+
+ @Test
+ void tokenize_coversEveryCharacter_ofBlankAndEmptyLines() throws Exception {
+ IGrammar grammar = loadGrammar("source.json", "json.json");
+
+ assertTiles(grammar, "{\r\n\r\n \"string\" :
\"metadata\"\r\n\r\n}\r\n");
+ }
+
+ /**
+ * What {@link
org.eclipse.jface.text.rules.DefaultDamagerRepairer#createPresentation} does
with a
+ * token stream: neighbouring tokens that share a TextAttribute become one
style range whose
+ * length is the sum of the token lengths, the offsets of the merged tokens
are never consulted.
+ * Returns the resulting {offset, end} pairs.
+ */
+ private static List<int[]> mergeLikeJFace(
+ List<ContentEditorTm4eSupport.ScopedRange> ranges,
+ java.util.function.Function<ContentEditorTm4eSupport.ScopedRange,
String> attribute) {
+ List<int[]> styleRanges = new ArrayList<>();
+ String lastAttribute = null;
+ int start = 0;
+ int length = 0;
+ for (ContentEditorTm4eSupport.ScopedRange range : ranges) {
+ String current = attribute.apply(range);
+ if (current.equals(lastAttribute)) {
+ length += range.length();
+ } else {
+ if (lastAttribute != null) {
+ styleRanges.add(new int[] {start, start + length});
+ }
+ lastAttribute = current;
+ start = range.offset();
+ length = range.length();
+ }
+ }
+ if (lastAttribute != null) {
+ styleRanges.add(new int[] {start, start + length});
+ }
+ return styleRanges;
+ }
+
+ @Test
+ void mergedStyleRuns_endWhereTheirTextEnds() throws Exception {
+ IGrammar grammar = loadGrammar("text.html.markdown", "markdown.json");
+ // Three heading lines in a row: every token on them carries
markup.heading, so JFace merges
+ // them into one style range. Each line delimiter it swallows used to cut
a character off the
+ // end of that range, which is what made the last heading lose its colour
(issue #7971).
+ String text = "# h1\n## h2\n### h3\n";
+
+ List<int[]> styleRanges =
+ mergeLikeJFace(
+ ContentEditorTm4eSupport.tokenize(grammar, text, 0, text.length()),
+ range ->
+ String.join(" ", range.scopes()).contains("markup.heading") ?
"heading" : "other");
+
+ int endOfLastHeading = text.indexOf("### h3") + "### h3".length();
+ assertTrue(
+ styleRanges.stream().anyMatch(r -> r[1] == endOfLastHeading),
+ "no style range ends at the end of '### h3' (offset "
+ + endOfLastHeading
+ + "), ranges: "
+ + styleRanges.stream().map(r -> r[0] + ".." + r[1]).toList());
+ }
+
+ @Test
+ void tokenize_appliesOnlyToTheRequestedRange() throws Exception {
+ IGrammar grammar = loadGrammar("source.sql", "sql.json");
+ String text = "SELECT\r\n p.CODE\r\nfrom\r\n";
+ int lineTwo = text.indexOf(" p.CODE");
+ int lineTwoLength = " p.CODE".length();
+
+ List<ContentEditorTm4eSupport.ScopedRange> ranges =
+ ContentEditorTm4eSupport.tokenize(grammar, text, lineTwo,
lineTwoLength);
+
+ assertEquals(lineTwo, ranges.getFirst().offset(), "should start at the
requested offset");
+ ContentEditorTm4eSupport.ScopedRange last = ranges.getLast();
+ assertEquals(
+ lineTwo + lineTwoLength,
+ last.offset() + last.length(),
+ "should stop at the end of the requested range");
+ }
+
@Test
void scopeForLanguage_markdownAliases_returnTextHtmlMarkdown() {
assertEquals("text.html.markdown",
ContentEditorTm4eSupport.scopeForLanguage("markdown"));
