This is an automated email from the ASF dual-hosted git repository. andy pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/jena.git
commit f2ff83711ece6efb15e434827f4a2d9ff0af2a3e Author: Andy Seaborne <[email protected]> AuthorDate: Wed Nov 6 14:45:32 2024 +0000 Rename 'Tokenizer' as 'SimpleTokenizer' --- .../org/apache/jena/reasoner/rulesys/Rule.java | 6 +- .../util/{Tokenizer.java => SimpleTokenizer.java} | 4 +- .../main/java/org/apache/jena/util/Tokenizer.java | 140 +-------------------- .../org/apache/jena/util/TestPackage_util.java | 2 +- ...TestTokenizer.java => TestSimpleTokenizer.java} | 10 +- 5 files changed, 15 insertions(+), 147 deletions(-) diff --git a/jena-core/src/main/java/org/apache/jena/reasoner/rulesys/Rule.java b/jena-core/src/main/java/org/apache/jena/reasoner/rulesys/Rule.java index d30e3cf28e..3dbdb3181f 100755 --- a/jena-core/src/main/java/org/apache/jena/reasoner/rulesys/Rule.java +++ b/jena-core/src/main/java/org/apache/jena/reasoner/rulesys/Rule.java @@ -35,7 +35,7 @@ import org.apache.jena.shared.WrappedIOException ; import org.apache.jena.util.FileManager ; import org.apache.jena.util.FileUtils ; import org.apache.jena.util.PrintUtil ; -import org.apache.jena.util.Tokenizer ; +import org.apache.jena.util.SimpleTokenizer ; import org.slf4j.Logger ; import org.slf4j.LoggerFactory ; @@ -702,7 +702,7 @@ public class Rule implements ClauseEntry { public static class Parser { /** Tokenizer */ - private Tokenizer stream; + private SimpleTokenizer stream; /** Look ahead, null if none */ private String lookahead; @@ -738,7 +738,7 @@ public class Rule implements ClauseEntry { * @param source the string to be parsed */ Parser(String source,BuiltinRegistry registry) { - stream = new Tokenizer(source, "()[], \t\n\r", "'\"", true); + stream = new SimpleTokenizer(source, "()[], \t\n\r", "'\"", true); lookahead = null; this.registry=registry; } diff --git a/jena-core/src/main/java/org/apache/jena/util/Tokenizer.java b/jena-core/src/main/java/org/apache/jena/util/SimpleTokenizer.java similarity index 97% copy from jena-core/src/main/java/org/apache/jena/util/Tokenizer.java copy to jena-core/src/main/java/org/apache/jena/util/SimpleTokenizer.java index 61526bca75..7405e0176d 100644 --- a/jena-core/src/main/java/org/apache/jena/util/Tokenizer.java +++ b/jena-core/src/main/java/org/apache/jena/util/SimpleTokenizer.java @@ -24,7 +24,7 @@ import java.util.NoSuchElementException; * A tokenizer, similar to java's StringTokenizer but allows for quoted * character strings which can include other separators. */ -public class Tokenizer { +public class SimpleTokenizer { /** The string being parsed */ protected String source; @@ -63,7 +63,7 @@ public class Tokenizer { * @param literalDelim Literal string delimiters * @param returnDelims If true then delimiters should be returned as tokens */ - public Tokenizer(String str, String delim, String literalDelim, boolean returnDelims) { + public SimpleTokenizer(String str, String delim, String literalDelim, boolean returnDelims) { this.source = str; this.delim = delim; this.literalDelim = literalDelim; diff --git a/jena-core/src/main/java/org/apache/jena/util/Tokenizer.java b/jena-core/src/main/java/org/apache/jena/util/Tokenizer.java index 61526bca75..ce2e13aeb7 100644 --- a/jena-core/src/main/java/org/apache/jena/util/Tokenizer.java +++ b/jena-core/src/main/java/org/apache/jena/util/Tokenizer.java @@ -18,144 +18,12 @@ package org.apache.jena.util; -import java.util.NoSuchElementException; - /** - * A tokenizer, similar to java's StringTokenizer but allows for quoted - * character strings which can include other separators. + * @deprecated Use {@link SimpleTokenizer}. */ -public class Tokenizer { - - /** The string being parsed */ - protected String source; - - /** The index of the first unreturned char in source */ - protected int p; - - /** The set of delimiter characters */ - protected String delim; - - /** If true then delimiters should be returned as tokens */ - protected boolean returnDelims; - - /** Literal string delimiters */ - protected String literalDelim; - - /** The lex state */ - protected int state; - - /** A lookahead for tokens */ - protected String lookahead; - - /** State flag: normal parse */ - protected static final int NORMAL = 1; - - /** State flag: start of literal */ - protected static final int LITERAL_START = 2; - - /** State flag: end of literal */ - protected static final int LITERAL_END = 3; - - /** - * Constructor. - * @param str the source string to be parsed - * @param delim The set of delimiter characters - * @param literalDelim Literal string delimiters - * @param returnDelims If true then delimiters should be returned as tokens - */ +@Deprecated(forRemoval = true) +public class Tokenizer extends SimpleTokenizer { public Tokenizer(String str, String delim, String literalDelim, boolean returnDelims) { - this.source = str; - this.delim = delim; - this.literalDelim = literalDelim; - this.returnDelims = returnDelims; - p = 0; - state = NORMAL; - } - - /** - * Return the next token. - * @throws java.util.NoSuchElementException if there are no more tokens available - */ - public String nextToken() { - String result = null; - if (lookahead != null) { - result = lookahead; - lookahead = null; - } else { - result = getNextToken(); - } - if (result == null) { - throw new NoSuchElementException("No more elements in tokenized string"); - } - if (!returnDelims) { - if (result.length() == 1) { - char c = result.charAt(0); - if (delim.indexOf(c) != -1 || literalDelim.indexOf(c) != -1) { - return nextToken(); - } - } - } - return result; - } - - /** - * Test if there are more tokens which can be returned. - */ - public boolean hasMoreTokens() { - if (lookahead == null) lookahead = getNextToken(); - return lookahead != null; - } - - /** - * Find the next token which can either be a delimiter or a real token. - */ - private String getNextToken() { - if (p >= source.length()) { - return null; - } - switch(state) { - case NORMAL: - if (is(literalDelim)) { - state = LITERAL_START; - p++; - return source.substring(p-1, p); - } else if (is(delim)) { - p++; - return source.substring(p-1, p); - } - int start = p; - p++; - while (p < source.length() && ! is(delim)) p++; - return source.substring(start, p); - case LITERAL_START: - char delim = source.charAt(p-1); - StringBuilder literal = new StringBuilder(); - while (p < source.length()) { - char c = source.charAt(p); - if (c == '\\') { - p++; - if (p >= source.length()) break; - c = source.charAt(p); - } else { - if (c == delim) break; - } - literal.append(c); - p++; - } - state = LITERAL_END; - return literal.toString(); - case LITERAL_END: - state = NORMAL; - p++; - return source.substring(p-1, p); - } - return null; - } - - /** - * Returns true if the current character is contained in the given classification. - */ - private boolean is(String classification) { - return classification.indexOf(source.charAt(p)) != -1; + super(str, delim, literalDelim, returnDelims); } } diff --git a/jena-core/src/test/java/org/apache/jena/util/TestPackage_util.java b/jena-core/src/test/java/org/apache/jena/util/TestPackage_util.java index b25bef47c6..b40812a9f8 100644 --- a/jena-core/src/test/java/org/apache/jena/util/TestPackage_util.java +++ b/jena-core/src/test/java/org/apache/jena/util/TestPackage_util.java @@ -35,7 +35,7 @@ public class TestPackage_util extends TestSuite { /** Creates new TestPackage */ private TestPackage_util() { super( "util" ); - addTest( "TestTokenzier", TestTokenizer.suite()); + addTest( "TestSimpleTokenzier", TestSimpleTokenizer.suite()); addTest( "TestFileUtils", TestFileUtils.suite() ); addTest( "TestHashUtils", TestCollectionFactory.suite() ); addTest( "TestLocationMapper", TestLocationMapper.suite() ) ; diff --git a/jena-core/src/test/java/org/apache/jena/util/TestTokenizer.java b/jena-core/src/test/java/org/apache/jena/util/TestSimpleTokenizer.java similarity index 86% rename from jena-core/src/test/java/org/apache/jena/util/TestTokenizer.java rename to jena-core/src/test/java/org/apache/jena/util/TestSimpleTokenizer.java index 04db621530..4a6aa46b82 100644 --- a/jena-core/src/test/java/org/apache/jena/util/TestTokenizer.java +++ b/jena-core/src/test/java/org/apache/jena/util/TestSimpleTokenizer.java @@ -24,12 +24,12 @@ import junit.framework.TestSuite; /** * Test for the trivial tokenizer utility. */ -public class TestTokenizer extends TestCase { +public class TestSimpleTokenizer extends TestCase { /** * Boilerplate for junit */ - public TestTokenizer( String name ) { + public TestSimpleTokenizer( String name ) { super( name ); } @@ -38,14 +38,14 @@ public class TestTokenizer extends TestCase { * This is its own test suite */ public static TestSuite suite() { - return new TestSuite( TestTokenizer.class ); + return new TestSuite( TestSimpleTokenizer.class ); } /** * Test tokenizer on a basic example or two. */ public void testTokenizer() { - Tokenizer tokenizer = new Tokenizer("a(foo,bar) 'i am a literal' so there", "()[], \t\n\r'", "'", true); + SimpleTokenizer tokenizer = new SimpleTokenizer("a(foo,bar) 'i am a literal' so there", "()[], \t\n\r'", "'", true); assertEquals(tokenizer.nextToken(), "a"); assertEquals(tokenizer.nextToken(), "("); assertEquals(tokenizer.nextToken(), "foo"); @@ -63,7 +63,7 @@ public class TestTokenizer extends TestCase { assertEquals(tokenizer.nextToken(), "there"); assertTrue( ! tokenizer.hasMoreTokens()); - tokenizer = new Tokenizer("a(foo,bar) 'i am a literal' so there", "()[], \t\n\r'", "'", false); + tokenizer = new SimpleTokenizer("a(foo,bar) 'i am a literal' so there", "()[], \t\n\r'", "'", false); assertEquals(tokenizer.nextToken(), "a"); assertEquals(tokenizer.nextToken(), "foo"); assertEquals(tokenizer.nextToken(), "bar");
