This is an automated email from the ASF dual-hosted git repository. afs pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/jena.git
commit aaaf6ee5908876631c60168727bc4badce593c07 Author: Andy Seaborne <[email protected]> AuthorDate: Mon May 18 18:32:24 2026 +0100 GH-3931: No Surrogates - RIOT --- .../src/main/java/org/apache/jena/query/ARQ.java | 51 ++-- .../src/main/java/org/apache/jena/riot/RIOT.java | 5 + .../org/apache/jena/riot/tokens/TokenizerText.java | 121 +++------ ...nizerText.java => TokenizerTextSurrogates.java} | 35 ++- .../org/apache/jena/riot/lang/TestLangTurtle.java | 14 +- .../AbstractTestTokenizerTextSurrogates.java | 293 +++++++++++++++++++++ .../org/apache/jena/riot/tokens/TS_Tokens.java | 6 +- .../apache/jena/riot/tokens/TestTokenizerText.java | 257 +----------------- .../TestTokenizerTextAllowEscSurrogates.java | 80 ++++++ ...ens.java => TestTokenizerTextNoSurrogates.java} | 33 ++- .../sparql/expr/TestSPARQLKeywordFunctions.java | 2 +- .../main/java/org/apache/jena/atlas/lib/Chars.java | 7 + 12 files changed, 515 insertions(+), 389 deletions(-) diff --git a/jena-arq/src/main/java/org/apache/jena/query/ARQ.java b/jena-arq/src/main/java/org/apache/jena/query/ARQ.java index 8667faccab..9b6277e758 100644 --- a/jena-arq/src/main/java/org/apache/jena/query/ARQ.java +++ b/jena-arq/src/main/java/org/apache/jena/query/ARQ.java @@ -139,47 +139,38 @@ public class ARQ /** Prefix for ARQ-defined parameter names */ public static final String arqSymbolPrefix = "arq"; - /** Stick exactly to the spec. - */ + /** Stick exactly to the spec. */ public static final Symbol strictSPARQL = SystemARQ.allocSymbol("strictSPARQL"); - /** Controls bNode labels as <_:...> or not - - * that is a pseudo URIs. - * This does not affect [] or _:a bNodes as variables in queries. - */ - - public static final Symbol constantBNodeLabels = - SystemARQ.allocSymbol("constantBNodeLabels"); - - /** Enable built-in property functions - also called "magic properties". - * These are properties in triple patterns that need - * calculation, not matching. See ARQ documentation for more details. - * rdfs:member and http://jena.apache.org/ARQ/list#member are provided. + /** + * Controls bNode labels as <_:...> or not - that is a pseudo URIs. This + * does not affect [] or _:a bNodes as variables in queries. */ + public static final Symbol constantBNodeLabels = SystemARQ.allocSymbol("constantBNodeLabels"); - public static final Symbol enablePropertyFunctions = - SystemARQ.allocSymbol("enablePropertyFunctions"); - - /** Enable logging of execution timing. + /** + * Enable built-in property functions - also called "magic properties". These are + * properties in triple patterns that need calculation, not matching. See ARQ + * documentation for more details. rdfs:member and + * http://jena.apache.org/ARQ/list#member are provided. */ + public static final Symbol enablePropertyFunctions = SystemARQ.allocSymbol("enablePropertyFunctions"); - public static final Symbol enableExecutionTimeLogging = - SystemARQ.allocSymbol("enableExecutionTimeLogging"); + /** Enable logging of execution timing. */ + public static final Symbol enableExecutionTimeLogging = SystemARQ.allocSymbol("enableExecutionTimeLogging"); - /** If true, XML result sets written will contain the graph bNode label - * See also inputGraphBNodeLabels + /** + * If true, XML result sets written will contain the graph bNode label. + * See also {@link #inputGraphBNodeLabels}. */ + public static final Symbol outputGraphBNodeLabels = SystemARQ.allocSymbol("outputGraphBNodeLabels"); - public static final Symbol outputGraphBNodeLabels = - SystemARQ.allocSymbol("outputGraphBNodeLabels"); - - /** If true, XML result sets will use the bNode label in the result set itself. - * See also outputGraphBNodeLabels + /** + * If true, XML result sets will use the bNode label in the result set itself. + * See also {@link #outputGraphBNodeLabels}. */ - - public static final Symbol inputGraphBNodeLabels = - SystemARQ.allocSymbol("inputGraphBNodeLabels"); + public static final Symbol inputGraphBNodeLabels = SystemARQ.allocSymbol("inputGraphBNodeLabels"); /** Turn on processing of blank node labels in queries */ public static void enableBlankNodeResultLabels() { enableBlankNodeResultLabels(true); } diff --git a/jena-arq/src/main/java/org/apache/jena/riot/RIOT.java b/jena-arq/src/main/java/org/apache/jena/riot/RIOT.java index 6ff7eaa7c4..f61ae0858a 100644 --- a/jena-arq/src/main/java/org/apache/jena/riot/RIOT.java +++ b/jena-arq/src/main/java/org/apache/jena/riot/RIOT.java @@ -22,6 +22,7 @@ package org.apache.jena.riot ; import org.apache.jena.atlas.lib.Version; +import org.apache.jena.query.ARQ; import org.apache.jena.riot.resultset.ResultSetLang; import org.apache.jena.riot.system.StreamRDF; import org.apache.jena.riot.writer.DirectiveStyle; @@ -60,6 +61,10 @@ public class RIOT { return systemGlobalContext; } + public static boolean isStrictMode() { + return ARQ.isStrictMode(); + } + public static void init() { if ( initialized ) return ; diff --git a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java index 3792b9288d..962da824c2 100644 --- a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java +++ b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java @@ -32,6 +32,7 @@ import org.apache.jena.atlas.io.IO; import org.apache.jena.atlas.io.PeekReader; import org.apache.jena.atlas.lib.Chars; import org.apache.jena.atlas.lib.Hex; +import org.apache.jena.atlas.lib.InternalErrorException; import org.apache.jena.riot.RiotParseException; import org.apache.jena.riot.system.ErrorHandler; import org.apache.jena.riot.system.RiotChars; @@ -99,9 +100,6 @@ public final class TokenizerText implements Tokenizer // CHECK_CODEPOINTS = true; // CHECK_RDFSTRING = false; - // Default=true. Whether to check for legal codepoint (i.e. only high-low surrogate pairs) while building strings. - private static final boolean CHECK_CODEPOINTS = true; - // Default=false. Whether to check for legal RDF strings (no ill formed use of surrogates) after building strings (normally done during string parsing). private static final boolean CHECK_RDFSTRING = false; @@ -611,9 +609,7 @@ public final class TokenizerText implements Tokenizer // It should be possible to rename stringBuilder with no changes to the code anywhere outside these operations.s private final StringBuilder stringBuilder = new StringBuilder(200); - private static final int NO_CODEPOINT = '\u0000'; - - // -- Unicode sequences with the possibility of codepoints beyond U+FFFF + // -- Unicode sequences with the possibility of codepoints beyond U+FFFF /** * String with the possibility of a unicode surrogate or unicode escape. * @@ -627,9 +623,7 @@ public final class TokenizerText implements Tokenizer * Check terminates correctly and return string. * Pair with {@link #startStringU()} */ - private String finishStringU(int finalCodepoint) { - if ( finalCodepoint != NO_CODEPOINT ) - fatal("Bad unpaired surrogate at end of string"); + private String finishStringU() { return stringBuilder.toString(); } @@ -659,56 +653,25 @@ public final class TokenizerText implements Tokenizer private void deleteCharAt(int idx) { stringBuilder.deleteCharAt(idx); } /** Insert codepoint. */ - private int insertCodepoint(int previousCP,int ch) { + private void insertCodepoint(int ch) { if ( Character.charCount(ch) == 1 ) { char ch16 = (char)ch; // Safe, not truncating, because count = 1 - char rtn = 0; - if ( CHECK_CODEPOINTS ) - rtn = checkCodepoint((char)previousCP, ch16); insertCodepointDirect(ch16); - return rtn; - } else { - // Surrogate waiting? - if ( CHECK_CODEPOINTS && (previousCP != NO_CODEPOINT) ) - fatal("Lone surrogate"); - if ( !Character.isDefined(ch) && !Character.isSupplementaryCodePoint(ch) ) - fatal("Illegal codepoint: 0x%04X", ch); - // Only legal surrogate pairs at this point. - char[] chars = Character.toChars(ch); - stringBuilder.append(chars); - return NO_CODEPOINT; - } - } - - // Only high then low is allowed. - // Casting int to char is a 16 bit silent truncation (bytecode "i2c"). - private char checkCodepoint(char previousCP, char ch) { - if ( ! Character.isSurrogate(ch) ) { - if ( previousCP == NO_CODEPOINT ) - return NO_CODEPOINT; - fatal("Bad surrogate (high surrogate not followed by a low surrogate): 0x%04X", (int)previousCP); - } - // Surrogate. - if ( previousCP == NO_CODEPOINT ) { // Effectively: is previousCodePoint a high surrogate? - if ( Character.isHighSurrogate(ch) ) { - // Park it - return ch; - } - fatal("Bad surrogate (low surrogate not preceded by a high surrogate): 0x%04X", (int)ch); - } - // previousCodePoint != NO_CODEPOINT - // Previous is a high surrogate - - if ( Character.isLowSurrogate(ch) ) { - // high-low -- OK! Clear previous. - return NO_CODEPOINT; + return; } - fatal("Bad surrogate (high surrogate not followed by low surrogate): 0x%04X", (int)previousCP); - return NO_CODEPOINT; + if ( !Character.isDefined(ch) && !Character.isSupplementaryCodePoint(ch) ) + fatal("Illegal codepoint: 0x%04X", ch); + char[] chars = Character.toChars(ch); + stringBuilder.append(chars); } + // XXX // Insert codepoint, knowing that 'ch' is 16 bit and not a surrogate. private void insertCodepointDirect(int ch) { + if ( ch >= Character.MIN_SUPPLEMENTARY_CODE_POINT ) { + String msg = String .format("Internal error: Attempt to insert a supplementary codepoint directly 0x04X", ch); + throw new InternalErrorException(msg); + } insertCodepointDirect((char)ch); } @@ -725,7 +688,6 @@ public final class TokenizerText implements Tokenizer // [8] IRIREF ::= '<' ([^#x00-#x20<>"{}|^`\] | UCHAR)* '>' private String readIRI() { startStringU(); - int prevCP = NO_CODEPOINT; for (;;) { int ch = reader.readChar(); switch(ch) { @@ -737,7 +699,7 @@ public final class TokenizerText implements Tokenizer fatal("Broken IRI (CR): %s", currentString()); return null; case CH_GT: // Done! - String str = finishStringU(prevCP); + String str = finishStringU(); if ( CHECK_RDFSTRING ) checkRDFString(str); return str; @@ -769,7 +731,7 @@ public final class TokenizerText implements Tokenizer if ( ch <= 0x19 ) warning("Illegal character in IRI (control char 0x%02X): <%s[0x%02X]...>", ch, currentString(), ch); } - prevCP = insertCodepoint(prevCP, ch); + insertCodepoint(ch); } } @@ -836,8 +798,6 @@ public final class TokenizerText implements Tokenizer // RiotChars has isPNChars_U_N for ( PN_CHARS_U | [0-9] ) - int prevCP = NO_CODEPOINT; - // -- Test first character int ch = reader.peekChar(); if ( ch == EOF ) @@ -848,7 +808,7 @@ public final class TokenizerText implements Tokenizer if ( isLocalPart ) { if ( ch == CH_COLON ) { reader.readChar(); - prevCP = insertCodepoint(prevCP, ch); + insertCodepoint(ch); } else if ( ch == CH_PERCENT || ch == CH_RSLASH ) { // processPLX // read % or \ @@ -860,18 +820,18 @@ public final class TokenizerText implements Tokenizer if ( ch == REPLACEMENT ) warning("Unicode replacement character U+FFFD in prefixed name"); } - prevCP = insertCodepoint(prevCP, ch); + insertCodepoint(ch); reader.readChar(); } else { - finishStringU(prevCP); + finishStringU(); return ""; } } else { if ( !RiotChars.isPNCharsBase(ch) ) { - finishStringU(prevCP); + finishStringU(); return ""; } - prevCP = insertCodepoint(prevCP, ch); + insertCodepoint(ch); reader.readChar(); } // Done first character @@ -885,7 +845,6 @@ public final class TokenizerText implements Tokenizer if ( chDot != 0 ) insertCodepointDirect(chDot); processPLX(ch); - prevCP = NO_CODEPOINT; chDot = 0; continue; } @@ -915,7 +874,7 @@ public final class TokenizerText implements Tokenizer if ( ch == REPLACEMENT ) warning("Unicode replacement character U+FFFD in prefixed name"); } - prevCP = insertCodepoint(prevCP, ch); + insertCodepoint(ch); } else { // DOT - delay until next loop. chDot = ch; @@ -928,7 +887,7 @@ public final class TokenizerText implements Tokenizer if ( chDot == CH_DOT ) // Unread it. reader.pushbackChar(chDot); - return finishStringU(prevCP); + return finishStringU(); } // Process PLX (percent or character escape for a prefixed name) @@ -990,7 +949,6 @@ public final class TokenizerText implements Tokenizer // Reads the terminating delimiter. startStringU(); - int prevCP = NO_CODEPOINT; for (;;) { int ch = reader.readChar(); if ( WarnOnReplacmentCharInString ) { @@ -1004,7 +962,7 @@ public final class TokenizerText implements Tokenizer fatal("Broken token: %s", currentString()); else if ( ch == endCh ) { // Done! - String str = finishStringU(prevCP); + String str = finishStringU(); if ( CHECK_RDFSTRING ) checkRDFString(str); return str; @@ -1020,7 +978,7 @@ public final class TokenizerText implements Tokenizer else if ( ch == CH_RSLASH ) // Allow escaped replacement character. ch = readLiteralEscape(); - prevCP = insertCodepoint(prevCP, ch); + insertCodepoint(ch); } } @@ -1028,7 +986,6 @@ public final class TokenizerText implements Tokenizer // Assumes the 3 character starting delimiter has been read. // Reads the terminating delimiter. startStringU(); - int prevCP = NO_CODEPOINT; for (;;) { int ch = reader.readChar(); if ( WarnOnReplacmentCharInString ) { @@ -1040,7 +997,7 @@ public final class TokenizerText implements Tokenizer fatal("Broken long string"); } else if ( ch == quoteChar ) { if ( threeQuotes(quoteChar) ) { - String str = finishStringU(prevCP); + String str = finishStringU(); if ( CHECK_RDFSTRING ) checkRDFString(str); return str; @@ -1048,7 +1005,7 @@ public final class TokenizerText implements Tokenizer // quote, not triple. It is a normal character. } else if ( ch == CH_RSLASH ) ch = readLiteralEscape(); - prevCP = insertCodepoint(prevCP, ch); + insertCodepoint(ch); } } @@ -1126,8 +1083,6 @@ public final class TokenizerText implements Tokenizer private String readBlankNodeLabel() { startStringU(); - - int prevCP = NO_CODEPOINT; // First character. { int ch = reader.peekChar(); @@ -1143,7 +1098,7 @@ public final class TokenizerText implements Tokenizer if ( ch == REPLACEMENT ) warning("Unicode replacement character U+FFFD in blank node label"); } - prevCP = insertCodepoint(prevCP, ch); + insertCodepoint(ch); } // Remainder. DOT can't be last so do a delay on that. @@ -1162,7 +1117,6 @@ public final class TokenizerText implements Tokenizer if ( chDot != 0 ) { insertCodepointDirect(chDot); - prevCP = NO_CODEPOINT; chDot = 0; } @@ -1172,7 +1126,7 @@ public final class TokenizerText implements Tokenizer if ( ch == REPLACEMENT ) warning("Unicode replacement character U+FFFD in blank node label"); } - prevCP = insertCodepoint(prevCP, ch); + insertCodepoint(ch); } else // DOT - delay until next loop. chDot = ch; @@ -1184,7 +1138,7 @@ public final class TokenizerText implements Tokenizer // if ( ! seen ) // exception("Blank node label missing"); - return finishStringU(prevCP); + return finishStringU(); } /* @@ -1398,7 +1352,7 @@ public final class TokenizerText implements Tokenizer } else break; } - return finishStringU(NO_CODEPOINT).intern(); + return finishStringU().intern(); } // ASCII-only e.g. in lang tags. @@ -1497,11 +1451,12 @@ public final class TokenizerText implements Tokenizer // Either \-uXXXX or \-u{...} private final int readUnicode4OrDelimEscape() { int nextCh = reader.peekChar(); - if ( nextCh == CH_LBRACE ) { + if ( nextCh == CH_LBRACE ) return readUnicodeDelimitedHex(); - } else { - return readUnicode4Escape(); - } + int ch = readUnicode4Escape(); + if ( isSurrogate(ch) ) + fatal("Surrogate codepoint from Unicode \\u escape sequence: 0x%04X", ch); + return ch; } private final int readUnicodeDelimitedHex() { @@ -1530,6 +1485,8 @@ public final class TokenizerText implements Tokenizer } if ( n == 0 ) fatal("Empty delimited hex escape sequence"); + if ( isSurrogate(value) ) + fatal("Surrogate codepoint from delimited Unicode escape sequence: 0x%04X", value); return value; } @@ -1542,6 +1499,8 @@ public final class TokenizerText implements Tokenizer // Check > 0x10FFFF if ( ! Character.isValidCodePoint(ch8) ) fatal("Illegal code point from \\U sequence value: 0x%08X", ch8); + if ( isSurrogate(ch8) ) + fatal("Surrogate codepoint from Unicode \\U escape sequence: 0x%04X", ch8); return ch8; } diff --git a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerTextSurrogates.java similarity index 97% copy from jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java copy to jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerTextSurrogates.java index 3792b9288d..eb29663c3f 100644 --- a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java +++ b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerTextSurrogates.java @@ -33,15 +33,29 @@ import org.apache.jena.atlas.io.PeekReader; import org.apache.jena.atlas.lib.Chars; import org.apache.jena.atlas.lib.Hex; import org.apache.jena.riot.RiotParseException; +import org.apache.jena.riot.SysRIOT; import org.apache.jena.riot.system.ErrorHandler; +import org.apache.jena.riot.system.ErrorHandlerFactory; import org.apache.jena.riot.system.RiotChars; import org.apache.jena.sparql.ARQInternalErrorException; +import org.slf4j.Logger; + + +// TokenizerText from Jena 6.1.0 +// +// This version accepts surrogate pairs written using numeric escape sequences +// and rejects surrogates ill-formed using numeric escape sequences +// +// Changing back to this tokenizer will affect tests. + +// TokenizerText now rejects all surrogates inline with the RDF 1.2 clarifications. +// Jena has not written surrogates numeric escape sequences in any version. /** * Tokenizer for the Turtle family of syntaxes. * Supports additional tokens. */ -public final class TokenizerText implements Tokenizer +/*public*/ final class TokenizerTextSurrogates implements Tokenizer { // This class is performance critical. @@ -148,15 +162,24 @@ public final class TokenizerText implements Tokenizer private Token token = null; private boolean finished = false; - public static TokenizerTextBuilder create() { return new TokenizerTextBuilder(); } + // public static TokenizerTextBuilder create() { return new TokenizerTextBuilder(); } + // + // public static Tokenizer fromString(String string) { return create().fromString(string).build(); } - public static Tokenizer fromString(String string) { return create().fromString(string).build(); } + // For tests + public static Tokenizer fromString(String string) { + PeekReader reader = PeekReader.readString(string); + Logger LOG = SysRIOT.getLogger(); + // Default - errors are exceptions, warning are logged. + ErrorHandler errorHandlerDft = ErrorHandlerFactory.errorHandlerWarnOrExceptions(LOG); + return new TokenizerTextSurrogates(reader, false, errorHandlerDft); + } - /*package*/ static TokenizerText internal(PeekReader reader, boolean singleLineMode, ErrorHandler errorHandler) { - return new TokenizerText(reader, singleLineMode, errorHandler); + /*package*/ static TokenizerTextSurrogates internal(PeekReader reader, boolean singleLineMode, ErrorHandler errorHandler) { + return new TokenizerTextSurrogates(reader, singleLineMode, errorHandler); } - private TokenizerText(PeekReader reader, boolean singleLineMode, ErrorHandler errorHandler) { + private TokenizerTextSurrogates(PeekReader reader, boolean singleLineMode, ErrorHandler errorHandler) { this.reader = Objects.requireNonNull(reader, "PeekReader"); this.singleLineMode = singleLineMode; this.errorHandler = Objects.requireNonNull(errorHandler, "ErrorHandler"); diff --git a/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangTurtle.java b/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangTurtle.java index 55b2d635df..1f60db7628 100644 --- a/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangTurtle.java +++ b/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangTurtle.java @@ -298,12 +298,18 @@ public class TestLangTurtle @Test public void turtle_surrogate_1() { // escaped high, escaped low - parseOneTriple("<x:s> <x:p> '\\ud800\\udc00' . "); + // If accepting escaped surrogates. + // parseOneTriple("<x:s> <x:p> '\\ud800\\udc00' . "); + // If not accepting escaped surrogates + parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\ud800\\udc00' . ")); } @Test public void turtle_surrogate_2() { // escaped high, raw low - parseOneTriple("<x:s> <x:p> '\\ud800\udc00' . "); + // If accepting escaped surrogates. + // parseOneTriple("<x:s> <x:p> '\\ud800\udc00' . "); + // If not accepting escaped surrogates + parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\ud800\udc00' . ")); } // Compilation failure. (maven+openjdk - OK in Eclipse, and test correct) @@ -339,7 +345,7 @@ public class TestLangTurtle @Test public void turtle_bad_surrogate_5() { // Wrong way round: low-high - parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\uc800\\ud800' . ")); + parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\udc00\\ud800' . ")); } // Compilation failure. Can't write \ud800 @@ -352,7 +358,7 @@ public class TestLangTurtle @Test public void turtle_bad_surrogate_7() { // escaped low - raw high - parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\uc800\ud800' . ")); + parseException(ExFatal.class, ()->parseOneTriple("<x:s> <x:p> '\\ud800\ud800' . ")); } // No Formulae. Not trig. diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/AbstractTestTokenizerTextSurrogates.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/AbstractTestTokenizerTextSurrogates.java new file mode 100644 index 0000000000..475ea00c6e --- /dev/null +++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/AbstractTestTokenizerTextSurrogates.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.apache.jena.riot.tokens; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import org.apache.jena.riot.RiotException; + +public abstract class AbstractTestTokenizerTextSurrogates { + + abstract protected Tokenizer tokenizer(String string); + + // Classification of tests + + // Structurally bad. + abstract protected void surrogateBad(String string); + + // Validly encoded into escape sequences. + abstract protected void surrogateValidEsc(String string); + + protected final void surrogateTest(String string) { + Tokenizer tokenizer = tokenizer(string); + assertTrue(tokenizer.hasNext()); + Token t1 = tokenizer.next(); + assertFalse(tokenizer.hasNext()); + } + + // Literal surrogates, i.e. java escape used to put into the tokenizer input. + // These don't get to the tokenizer escape processing code. + // They should be as if the UTF-8 has supplemental codepoints. + protected void surrogateRawGood(String string) { + surrogateTest(string); + } + + // Literal surrogates, i.e. java escape used to put into the tokenizer input but illegal in some way. + protected final void surrogateRawBad(String string) { + assertThrows(RiotException.class, ()->surrogateTest(string)); + } + + // ---- + + // U+D800-U+DBFF is a high surrogate (first part of a pair) + // U+DC00-U+DFFF is a low surrogate (second part of a pair) + // so D800-DC00 is legal. + + @Test public void turtle_surrogate_pair_esc_esc_01() { + // escaped high, escaped low + surrogateValidEsc("'\\ud800\\udc00'"); + } + + @Test public void turtle_surrogate_pair_esc_esc_02() { + // escaped high, escaped low + surrogateValidEsc("'''\\ud800\\udc00'''"); + } + + @Test public void turtle_surrogate_pair_esc_esc_03() { + // escaped high, escaped low + surrogateValidEsc("<\\ud800\\udc00>"); + } + + @Test public void turtle_surrogate_pair_esc_raw_01() { + // escaped high, raw low + surrogateValidEsc("'\\ud800\udc00'"); + } + + @Test public void turtle_surrogate_pair_esc_raw_02() { + // escaped high, raw low + surrogateValidEsc("'''\\ud800\udc00'''"); + } + + @Test public void turtle_surrogate_pair_esc_raw_03() { + // escaped high, raw low + surrogateRawBad("<\\ud800\udc00>"); + } + + // Compilation failure - illegal escape character +// @Test public void turtle_surrogate_pair_raw_esc_01() { +// // raw high, escaped low +// surrogate("'\ud800\\udc00'"); +// } + + @Test public void turtle_surrogate_pair_raw_raw_01() { + // raw high, raw low + surrogateRawGood("'\ud800\udc00'"); + } + + @Test public void turtle_surrogate_pair_raw_raw_02() { + // raw high, raw low + surrogateRawGood("'''\ud800\udc00'''"); + } + + @Test public void turtle_surrogate_pair_raw_raw_03() { + // raw high, raw low + surrogateRawGood("<\ud800\udc00>"); + } + + // Blank nodes label allow unicode but not unicode escapes. + @Test public void turtle_surrogate_pair_raw_raw_04() { + // raw high, raw low + surrogateRawGood("_:b\ud800\udc00"); + } + + @Test public void turtle_surrogate_pair_raw_raw_05() { + surrogateRawGood("ns:\ud800\udc00"); + } + + @Test public void turtle_surrogate_pair_raw_raw_06() { + surrogateRawGood("\ud800\udc00:local"); + } + + @Test public void turtle_surrogate_pair_esc_esc_internal_01() { + // escaped high, escaped low + surrogateValidEsc("'a\\ud800\\udc00x'"); + } + + @Test public void turtle_surrogate_pair_esc_esc_internal_02() { + // escaped high, escaped low + surrogateValidEsc("'''a\\ud800\\udc00x'''"); + } + + @Test public void turtle_surrogate_pair_esc_esc_internal_03() { + // escaped high, escaped low + surrogateValidEsc("<a\\ud800\\udc00x>"); + } + + @Test public void turtle_surrogate_pair_esc_raw_internal_01() { + // escaped high, raw low + surrogateRawBad("'z\\ud800\udc00z'"); + } + + @Test public void turtle_surrogate_pair_esc_raw_internal_02() { + // escaped high, raw low + surrogateRawBad("'''z\\ud800\udc00z'''"); + } + + @Test public void turtle_surrogate_pair_esc_raw_internal_03() { + // escaped high, raw low + surrogateRawBad("<z\\ud800\udc00z>"); + } + + // Compilation failure - illegal escape character +// @Test public void turtle_surrogate_pair_raw_esc() { +// // raw high, escaped low +// surrogate("'a\ud800\\udc00'z"); +// } + + // Java, not RDF, surrogates. + @Test public void turtle_surrogate_pair_raw_raw_internal_01() { + // raw high, raw low + surrogateRawGood("'a\ud800\udc00z'"); + } + + @Test public void turtle_surrogate_pair_raw_raw_internal_02() { + // raw high, raw low + surrogateRawGood("'''a\ud800\udc00z'''"); + } + + // XXX Overlap + @Test public void turtle_surrogate_pair_raw_raw_internal_03() { + // raw high, raw low + surrogateRawGood("<a\ud800\udc00z>"); + } + + @Test public void turtle_surrogate_pair_raw_raw_internal_04() { + // raw high, raw low + surrogateRawGood("_:ba\ud800\udc00z"); + } + + @Test public void turtle_surrogate_pair_raw_raw__internal05() { + surrogateRawGood("ns:x\ud800\udc00y"); + } + + @Test public void turtle_surrogate_pair_raw_raw__internal06() { + surrogateRawGood("x\ud800\udc00y:local"); + } + + @Test + public void turtle_bad_surrogate_01() { + surrogateBad("'\\ud800'"); + } + + @Test + public void turtle_bad_surrogate_01a() { + surrogateBad("'''\\ud800''''"); + } + + @Test + public void turtle_bad_surrogate_02() { + surrogateBad("'a\\ud800z'"); + } + + @Test + public void turtle_bad_surrogate_02a() { + surrogateBad("'''a\\ud800z'''"); + } + + @Test + public void turtle_bad_surrogate_02b() { + surrogateBad("<a\\ud800z>"); + } + + @Test + public void turtle_bad_surrogate_03() { + surrogateBad("'\\udfff'"); + } + + @Test + public void turtle_bad_surrogate_04() { + surrogateBad("'a\\udfffz'"); + } + + @Test + public void turtle_bad_surrogate_05() { + surrogateBad("'\\U0000d800'"); + } + + @Test + public void turtle_bad_surrogate_06() { + surrogateBad("'a\\U0000d800z'"); + } + + @Test + public void turtle_bad_surrogate_07() { + surrogateBad("'\\U0000dfff'"); + } + + @Test + public void turtle_bad_surrogate_08() { + surrogateBad("'a\\U0000dfffz'"); + } + + @Test + public void turtle_bad_surrogate_09() { + // Wrong way round: low-high; + surrogateBad("'\\udc00\\ud800'"); + } + + @Test + public void turtle_bad_surrogate_10() { + // Wrong way round: low-high + surrogateBad("'a\\udc00\ud800z'"); + } + + // Compilation failure - illegal escape character +// @Test +// public void turtle_bad_surrogate_11() { +// // raw low - escaped high +// assertThrows(RiotParseException.class, ()-> +// surrogate("'\ud800\\ud800'"); +// } +// +// @Test +// public void turtle_bad_surrogate_12() { +// // raw low - escaped high +// assertThrows(RiotParseException.class, ()-> +// surrogate("'a\ud800\\ud800z'"); +// } + + @Test + public void turtle_bad_surrogate_13() { + // escaped low - raw high + surrogateBad("'\\udc00\ud800'"); + } + + @Test + public void turtle_bad_surrogate_14() { + // escaped low - raw high + surrogateBad("'a\\dc00\ud800z'"); + } +} diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java index 9d4f87226c..044468dff6 100644 --- a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java +++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java @@ -26,8 +26,10 @@ import org.junit.platform.suite.api.Suite; @Suite @SelectClasses({ - TestTokenizerText.class - , TestTokenForNode.class + TestTokenForNode.class + , TestTokenizerText.class + , TestTokenizerTextNoSurrogates.class + , TestTokenizerTextAllowEscSurrogates.class }) public class TS_Tokens diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java index 64af6b8ad7..c82b2aca52 100644 --- a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java +++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java @@ -39,6 +39,7 @@ import org.apache.jena.riot.system.ErrorHandlerFactory.ErrorHandlerRecorder; import org.apache.jena.sparql.ARQConstants; public class TestTokenizerText { + // Surrogate related tests in TestTokenizerTextNoSurrogates. private static Tokenizer tokenizer(String string) { return tokenizer(string, false); @@ -1380,262 +1381,6 @@ public class TestTokenizerText { assertFalse(tokenizer.hasNext()); } - // U+D800-U+DBFF is a high surrogate (first part of a pair) - // U+DC00-U+DFFF is a low surrogate (second part of a pair) - // so D800-DC00 is legal. - - @Test public void turtle_surrogate_pair_esc_esc_01() { - // escaped high, escaped low - surrogate("'\\ud800\\udc00'"); - } - - @Test public void turtle_surrogate_pair_esc_esc_02() { - // escaped high, escaped low - surrogate("'''\\ud800\\udc00'''"); - } - - @Test public void turtle_surrogate_pair_esc_esc_03() { - // escaped high, escaped low - surrogate("<\\ud800\\udc00>"); - } - - @Test public void turtle_surrogate_pair_esc_raw_01() { - // escaped high, raw low - surrogate("'\\ud800\udc00'"); - } - - @Test public void turtle_surrogate_pair_esc_raw_02() { - // escaped high, raw low - surrogate("'''\\ud800\udc00'''"); - } - @Test public void turtle_surrogate_pair_esc_raw_03() { - // escaped high, raw low - surrogate("<\\ud800\udc00>"); - } - @Test public void turtle_surrogate_pair_esc_raw_04() { - // escaped high, raw low - surrogate("_:b\ud800\udc00"); - } - - // Compilation failure - illegal escape character -// @Test public void turtle_surrogate_pair_raw_esc_01() { -// // raw high, escaped low -// surrogate("'\ud800\\udc00'"); -// } - - @Test public void turtle_surrogate_pair_raw_raw_01() { - // raw high, raw low - surrogate("'\ud800\udc00'"); - } - - @Test public void turtle_surrogate_pair_raw_raw_02() { - // raw high, raw low - surrogate("'''\ud800\udc00'''"); - } - - @Test public void turtle_surrogate_pair_raw_raw_03() { - // raw high, raw low - surrogate("<\ud800\udc00>"); - } - - // Blank nodes label allow unicode but not unicode escapes. - @Test public void turtle_surrogate_pair_raw_raw_04() { - // raw high, raw low - surrogate("_:b\ud800\udc00"); - } - - @Test public void turtle_surrogate_pair_raw_raw_05() { - // escaped high, escaped low - surrogate("ns:\ud800\udc00"); - } - - @Test public void turtle_surrogate_pair_raw_raw_06() { - // escaped high, escaped low - surrogate("\ud800\udc00:local"); - } - - @Test public void turtle_surrogate_pair_esc_esc_internal_01() { - // escaped high, escaped low - surrogate("'a\\ud800\\udc00x'"); - } - - @Test public void turtle_surrogate_pair_esc_esc_internal_02() { - // escaped high, escaped low - surrogate("'''a\\ud800\\udc00x'''"); - } - - @Test public void turtle_surrogate_pair_esc_esc_internal_03() { - // escaped high, escaped low - surrogate("<a\\ud800\\udc00x>"); - } - - @Test public void turtle_surrogate_pair_esc_raw_internal_01() { - // escaped high, raw low - surrogate("'z\\ud800\udc00z'"); - } - - @Test public void turtle_surrogate_pair_esc_raw_internal_02() { - // escaped high, raw low - surrogate("'''z\\ud800\udc00z'''"); - } - - @Test public void turtle_surrogate_pair_esc_raw_internal_03() { - // escaped high, raw low - surrogate("<z\\ud800\udc00z>"); - } - - // Compilation failure - illegal escape character -// @Test public void turtle_surrogate_pair_raw_esc() { -// // raw high, escaped low -// surrogate("'a\ud800\\udc00'z"); -// } - - @Test public void turtle_surrogate_pair_raw_raw_internal_01() { - // raw high, raw low - surrogate("'a\ud800\udc00z'"); - } - - @Test public void turtle_surrogate_pair_raw_raw_internal_02() { - // raw high, raw low - surrogate("'''a\ud800\udc00z'''"); - } - - @Test public void turtle_surrogate_pair_raw_raw_internal_03() { - // raw high, raw low - surrogate("<a\ud800\udc00z>"); - } - - @Test public void turtle_surrogate_pair_raw_raw_internal_04() { - // raw high, raw low - surrogate("_:ba\ud800\udc00z"); - } - - @Test public void turtle_surrogate_pair_raw_raw__internal05() { - // escaped high, escaped low - surrogate("ns:x\ud800\udc00y"); - } - - @Test public void turtle_surrogate_pair_raw_raw__internal06() { - // escaped high, escaped low - surrogate("x\ud800\udc00y:local"); - } - - @Test - public void turtle_bad_surrogate_01() { - assertThrows(RiotParseException.class, ()-> - surrogate("'\\ud800'")); - } - - @Test - public void turtle_bad_surrogate_01a() { - assertThrows(RiotParseException.class, ()-> - surrogate("'''\\ud800''''")); - } - - @Test - public void turtle_bad_surrogate_02() { - assertThrows(RiotParseException.class, ()-> - surrogate("'a\\ud800z'")); - } - - @Test - public void turtle_bad_surrogate_02a() { - assertThrows(RiotParseException.class, ()-> - surrogate("'''a\\ud800z'''")); - } - - @Test - public void turtle_bad_surrogate_02b() { - assertThrows(RiotParseException.class, ()-> - surrogate("<a\\ud800z>")); - } - - @Test - public void turtle_bad_surrogate_03() { - assertThrows(RiotParseException.class, ()-> - surrogate("'\\udfff'")); - } - - @Test - public void turtle_bad_surrogate_04() { - assertThrows(RiotParseException.class, ()-> - surrogate("'a\\udfffz'")); - } - - @Test - public void turtle_bad_surrogate_05() { - assertThrows(RiotParseException.class, ()-> - surrogate("'\\U0000d800'")); - } - - @Test - public void turtle_bad_surrogate_06() { - assertThrows(RiotParseException.class, ()-> - surrogate("'a\\U0000d800z'")); - } - - @Test - public void turtle_bad_surrogate_07() { - assertThrows(RiotParseException.class, ()-> - surrogate("'\\U0000dfff'")); - } - - @Test - public void turtle_bad_surrogate_08() { - assertThrows(RiotParseException.class, ()-> - surrogate("'a\\U0000dfffz'")); - } - - @Test - public void turtle_bad_surrogate_09() { - // Wrong way round: low-hig); - assertThrows(RiotParseException.class, ()-> - surrogate("'\\uc800\\ud800'")); - } - - @Test - public void turtle_bad_surrogate_10() { - assertThrows(RiotParseException.class, ()-> - // Wrong way round: low-high - surrogate("'a\\uc800\\ud800z'")); - } - - // Compilation failure - illegal escape character -// @Test -// public void turtle_bad_surrogate_11() { -// // raw low - escaped high -// assertThrows(RiotParseException.class, ()-> -// surrogate("'\ud800\\ud800'")); -// } -// -// @Test -// public void turtle_bad_surrogate_12() { -// // raw low - escaped high -// assertThrows(RiotParseException.class, ()-> -// surrogate("'a\ud800\\ud800z'")); -// } - - @Test - public void turtle_bad_surrogate_13() { - // escaped low - raw high - assertThrows(RiotParseException.class, ()-> - surrogate("'\\uc800\ud800'")); - } - - @Test - public void turtle_bad_surrogate_14() { - // escaped low - raw high - assertThrows(RiotParseException.class, ()-> - surrogate("'a\\uc800\ud800z'")); - } - - private void surrogate(String string) { - Tokenizer tokenizer = tokenizer(string); - assertTrue(tokenizer.hasNext()); - tokenizer.next(); - assertFalse(tokenizer.hasNext()); - } - // \-u{...} style Unicode escapes @Test public void unescape_unicode_20() { test_unesc_unicode("\\u{41}", "A") ; } @Test public void unescape_unicode_21() { test_unesc_unicode("\\u{000000}", "\u0000") ; } diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextAllowEscSurrogates.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextAllowEscSurrogates.java new file mode 100644 index 0000000000..13196c5176 --- /dev/null +++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextAllowEscSurrogates.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.apache.jena.riot.tokens; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import org.apache.jena.riot.RiotException; + +// Tests for the variant of TokenizerText that handles escaped surrogates +public class TestTokenizerTextAllowEscSurrogates extends AbstractTestTokenizerTextSurrogates { + + @Override + protected Tokenizer tokenizer(String string) { + return TokenizerTextSurrogates.fromString(string); + } + + // Structurally bad. + @Override + protected void surrogateBad(String string) { + assertThrows(RiotException.class, ()->surrogateTest(string)); + } + + // Validly encoded into escape sequences. + // Jena now rejects all surrogates, via any route. + // Testing the legacy TokenizerTextSurrogates + @Override + protected void surrogateValidEsc(String string) { + surrogateTest(string); + } + + // These are "surrogateRawBad" in the "no surrogates" test suite + // but valid (and junk) if accepting surrogates. + // 6.1.0 compatibility. + // The esc-raw case is wrong in TokenizerTextSurrogates.checkCodepoint. + // It inserts a high surrogate early which then gets a real raw low surrogate and "works". + // Probably not what was intended! + + @Override + @Test public void turtle_surrogate_pair_esc_raw_internal_01() { + //surrogateValidEsc("'z\\ud800\udc00z'"); + surrogateValidEsc("'\\ud800\udc00z'"); + } + + @Override + @Test public void turtle_surrogate_pair_esc_raw_internal_02() { + surrogateValidEsc("'''z\\ud800\udc00z'''"); + } + + @Override + @Test public void turtle_surrogate_pair_esc_raw_internal_03() { + // escaped high, raw low + surrogateValidEsc("<z\\ud800\udc00z>"); + } + + @Override + @Test public void turtle_surrogate_pair_esc_raw_03() { + surrogateValidEsc("<\\ud800\udc00>"); + } +} diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextNoSurrogates.java similarity index 51% copy from jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java copy to jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextNoSurrogates.java index 9d4f87226c..87a6748155 100644 --- a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TS_Tokens.java +++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerTextNoSurrogates.java @@ -21,15 +21,30 @@ package org.apache.jena.riot.tokens; -import org.junit.platform.suite.api.SelectClasses; -import org.junit.platform.suite.api.Suite; +import static org.junit.jupiter.api.Assertions.assertThrows; -@Suite -@SelectClasses({ - TestTokenizerText.class - , TestTokenForNode.class -}) +import org.apache.jena.riot.RiotException; -public class TS_Tokens -{ } +// Tests for the variant of TokenizerText that handles escaped surrogates +public class TestTokenizerTextNoSurrogates extends AbstractTestTokenizerTextSurrogates { + @Override + protected Tokenizer tokenizer(String string) { + return TokenizerText.fromString(string); + } + + // Reject all use of surrogates. + + // Structurally bad. + @Override + protected void surrogateBad(String string) { + assertThrows(RiotException.class, ()->surrogateTest(string)); + } + + // Validly encoded into escape sequences. + // Jena now rejects all surrogates, via any route. + @Override + protected void surrogateValidEsc(String string) { + assertThrows(RiotException.class, ()->surrogateTest(string)); + } +} diff --git a/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java b/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java index 3ec13e2e9b..4502fe4639 100644 --- a/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java +++ b/jena-arq/src/test/java/org/apache/jena/sparql/expr/TestSPARQLKeywordFunctions.java @@ -205,7 +205,7 @@ public class TestSPARQLKeywordFunctions @Test public void substr_30() { test("substr('\\uD83D\\uDC6A', 1)", "'\\uD83D\\uDC6A'"); } // Same using Java string escapes. @Test public void substr_30b() { test("substr('\uD83D\uDC6A', 1)", "'\uD83D\uDC6A'"); } - @Test public void substr_31() { test("substr('\\uD83D\\uDC6A', 2)", kwEmptyString); } + @Test public void substr_31() { test("substr('\\uD83D\\uDC6A', 2)", kwEmptyString); } @Test public void substr_32() { test("substr('ABC\\uD83D\\uDC6ADEF', 4, 1)", "'\\uD83D\\uDC6A'"); } @Test public void substr_33() { test("substr('\\uD83D\\uDC6A!', -1, 3)", "'\\uD83D\\uDC6A'"); } diff --git a/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java b/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java index b77861901a..0ef6b9faff 100644 --- a/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java +++ b/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java @@ -180,6 +180,13 @@ public class Chars // REPLACEMENT CHARACTER public static final char REPLACEMENT = 0xFFFD ; + /** Test for a surrogate codepoint */ + public static boolean isSurrogate(int ch) { + // The Character class only has char tests. + // The range D800-DFFF (high/leading: D800-DBFF, low/trailing: DC00-DFFF) + return ch >= Character.MIN_HIGH_SURROGATE && ch <= Character.MAX_LOW_SURROGATE; + } + // "Not a character" - a Unicode non-character codepoint that is not legal in UTF-8. public static final char NotACharacter = 0xFFFF ; // Detect byte order by contrast (BOM reversed) - a Unicode non-character codepoint that is not legal in UTF-8.
