JENA-1523: Allow internal variables names in variable parsing.
Project: http://git-wip-us.apache.org/repos/asf/jena/repo Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/991ec09c Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/991ec09c Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/991ec09c Branch: refs/heads/master Commit: 991ec09cb1ac24c4c3b3e9d440b426045d2207be Parents: db818ea Author: Andy Seaborne <a...@apache.org> Authored: Fri Apr 13 17:53:48 2018 +0100 Committer: Andy Seaborne <a...@apache.org> Committed: Fri Apr 13 18:21:59 2018 +0100 ---------------------------------------------------------------------- .../org/apache/jena/riot/tokens/TokenizerText.java | 11 ++++++++--- .../org/apache/jena/riot/tokens/TestTokenizer.java | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jena/blob/991ec09c/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java ---------------------------------------------------------------------- diff --git a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java index d3cd80c..9f92fe9 100644 --- a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java +++ b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java @@ -739,7 +739,13 @@ public final class TokenizerText implements Tokenizer return readCharsAnd(leadingDigitAllowed, leadingSignAllowed, extraCharsWord, false); } - static private char[] extraCharsVar = new char[]{'_', '.', '-', '?', '@', '+'}; + // This array adds the other characters that can occurs in an internal variable name. + // Variables can be created with SPARQL-illegal syntax to ensure they do not clash with + // variables in the query from the application. + // See ARQConstants. + // allocVarAnonMarker, allocVarMarker, globalVar, allocVarBNodeToVar, allocVarScopeHiding + // but this set is wider and matches anywhere in the name after the first '?'. + static private char[] extraCharsVar = new char[]{'_', '.', '-', '?', '@', '+', '/', '~'}; private String readVarName() { return readCharsAnd(true, true, extraCharsVar, true); @@ -747,8 +753,7 @@ public final class TokenizerText implements Tokenizer // See also readBlankNodeLabel - private String readCharsAnd(boolean leadingDigitAllowed, boolean leadingSignAllowed, char[] extraChars, - boolean allowFinalDot) { + private String readCharsAnd(boolean leadingDigitAllowed, boolean leadingSignAllowed, char[] extraChars, boolean allowFinalDot) { stringBuilder.setLength(0); int idx = 0; if ( !leadingDigitAllowed ) { http://git-wip-us.apache.org/repos/asf/jena/blob/991ec09c/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizer.java ---------------------------------------------------------------------- diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizer.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizer.java index 4baf1f2..f7ae2d1 100644 --- a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizer.java +++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizer.java @@ -655,6 +655,21 @@ public class TestTokenizer extends BaseTest { } @Test + public void tokenUnit_var7() { + tokenizeAndTestExact("?" + ARQConstants.allocVarScopeHiding + "0", TokenType.VAR, ARQConstants.allocVarScopeHiding + "0") ; + } + + @Test + public void tokenUnit_var8() { + tokenizeAndTestExact("?" + ARQConstants.allocVarAnonMarker + "0", TokenType.VAR, ARQConstants.allocVarAnonMarker + "0") ; + } + + @Test + public void tokenUnit_var9() { + tokenizeAndTestExact("?" + ARQConstants.allocVarBNodeToVar + "ABC", TokenType.VAR, ARQConstants.allocVarBNodeToVar + "ABC") ; + } + + @Test public void tokenUnit_hex1() { tokenizeAndTestExact("0xABC", TokenType.HEX, "0xABC") ; }