This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch CAMEL-25015-tokenize-literal-token in repository https://gitbox.apache.org/repos/asf/camel.git
commit 65c89ed472095ed448d2fc97e8a7d841634b2633 Author: Claus Ibsen <[email protected]> AuthorDate: Sun Nov 1 20:01:13 2026 +0100 CAMEL-25015: camel-core - Tokenize language: the token is literal text unless regex is true The token was always compiled as a regular expression, so tokenize("|") split every character, tokenize(".") gave empty parts and tokenize("*") failed with a PatternSyntaxException. The token is now quoted unless regex is true, and so is the end token of the pair mode, which replaces the escaping of [ and ] by hand. The group delimiter of a token with a Simple function is the token, not the quoted pattern. In the XML and YAML DSL the escapes \n, \r and \t in a token that is not a regular expression are turned into the characters, so token="\r\n" keeps working. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../camel/catalog/docs/tokenize-language.adoc | 5 + .../modules/languages/pages/tokenize-language.adoc | 5 + .../language/TokenizerExpressionReifier.java | 20 +-- .../org/apache/camel/language/TokenizerTest.java | 8 +- .../tokenizer/TokenizeLiteralTokenTest.java | 139 +++++++++++++++++++++ .../apache/camel/support/GroupTokenIterator.java | 14 ++- .../camel/support/builder/ExpressionBuilder.java | 9 +- .../builder/TokenPairExpressionIterator.java | 22 +--- .../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 17 +++ 9 files changed, 208 insertions(+), 31 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/tokenize-language.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/tokenize-language.adoc index 729cb7db7f83..2891d7f124a0 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/tokenize-language.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/tokenize-language.adoc @@ -37,6 +37,11 @@ then forward each piece to direct:b: IMPORTANT: When using new-line (`\n`) as tokenizer, then XML and YAML DSL must use the `token` option\ as done in the following example. +The token is matched as literal text, so a delimiter such as `|`, `.` or `*` splits at that character. +To use a regular expression as token, such as `\s+`, set the `regex` option to `true`. +In the XML and YAML DSL, the escapes `\n`, `\r` and `\t` in a token are the new line, carriage return +and tab characters, so `\r\n` splits Windows lines. + [tabs] ==== diff --git a/core/camel-core-languages/src/main/docs/modules/languages/pages/tokenize-language.adoc b/core/camel-core-languages/src/main/docs/modules/languages/pages/tokenize-language.adoc index 729cb7db7f83..2891d7f124a0 100644 --- a/core/camel-core-languages/src/main/docs/modules/languages/pages/tokenize-language.adoc +++ b/core/camel-core-languages/src/main/docs/modules/languages/pages/tokenize-language.adoc @@ -37,6 +37,11 @@ then forward each piece to direct:b: IMPORTANT: When using new-line (`\n`) as tokenizer, then XML and YAML DSL must use the `token` option\ as done in the following example. +The token is matched as literal text, so a delimiter such as `|`, `.` or `*` splits at that character. +To use a regular expression as token, such as `\s+`, set the `regex` option to `true`. +In the XML and YAML DSL, the escapes `\n`, `\r` and `\t` in a token are the new line, carriage return +and tab characters, so `\r\n` splits Windows lines. + [tabs] ==== diff --git a/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/language/TokenizerExpressionReifier.java b/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/language/TokenizerExpressionReifier.java index e1c7bd553418..47b40bb5b537 100644 --- a/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/language/TokenizerExpressionReifier.java +++ b/core/camel-core-reifier/src/main/java/org/apache/camel/reifier/language/TokenizerExpressionReifier.java @@ -30,17 +30,23 @@ public class TokenizerExpressionReifier extends SingleInputTypedExpressionReifie Object[] properties = new Object[11]; properties[0] = asResultType(); properties[1] = parseString(definition.getSource()); - // special for new line tokens, if defined from XML then its 2 - // characters, so we replace that back to a single char - String token = definition.getToken(); - if (token != null && token.startsWith("\\n")) { - token = '\n' + token.substring(2); + Boolean regex = parseBoolean(definition.getRegex()); + // special for new line, carriage return and tab in tokens, if defined from XML then they are 2 + // characters, so we replace them back to a single char. A regular expression understands them + // as they are, so only a leading new line is replaced there + String token = parseString(definition.getToken()); + if (token != null && Boolean.TRUE.equals(regex)) { + if (token.startsWith("\\n")) { + token = '\n' + token.substring(2); + } + } else if (token != null) { + token = token.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t"); } - properties[2] = parseString(token); + properties[2] = token; properties[3] = parseString(definition.getEndToken()); properties[4] = parseString(definition.getInheritNamespaceTagName()); properties[5] = parseString(definition.getGroupDelimiter()); - properties[6] = parseBoolean(definition.getRegex()); + properties[6] = regex; properties[7] = parseBoolean(definition.getXml()); properties[8] = parseBoolean(definition.getIncludeTokens()); properties[9] = parseString(definition.getGroup()); diff --git a/core/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java b/core/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java index 559b6d0b46a7..365b1bfd1138 100644 --- a/core/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java +++ b/core/camel-core/src/test/java/org/apache/camel/language/TokenizerTest.java @@ -40,9 +40,13 @@ public class TokenizerTest extends ExchangeTestSupport { } Expression tokenize(String headerName, String token) { + return tokenize(headerName, token, false); + } + + Expression tokenize(String headerName, String token, boolean regex) { TokenizeLanguage language = new TokenizeLanguage(); language.setCamelContext(context); - return language.createExpression(null, new Object[] { null, "header:" + headerName, token }); + return language.createExpression(null, new Object[] { null, "header:" + headerName, token, null, null, null, regex }); } Expression tokenizePair(String startToken, String endToken, boolean includeTokens) { @@ -119,7 +123,7 @@ public class TokenizerTest extends ExchangeTestSupport { @Test public void testTokenizeHeaderRegEx() { - Expression exp = tokenize("quote", "(\\W+)\\s*"); + Expression exp = tokenize("quote", "(\\W+)\\s*", true); exp.init(context); exchange.getIn().setHeader("quote", "Camel rocks"); diff --git a/core/camel-core/src/test/java/org/apache/camel/language/tokenizer/TokenizeLiteralTokenTest.java b/core/camel-core/src/test/java/org/apache/camel/language/tokenizer/TokenizeLiteralTokenTest.java new file mode 100644 index 000000000000..4cdec4a2870b --- /dev/null +++ b/core/camel-core/src/test/java/org/apache/camel/language/tokenizer/TokenizeLiteralTokenTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.language.tokenizer; + +import java.util.List; + +import org.apache.camel.ContextTestSupport; +import org.apache.camel.Exchange; +import org.apache.camel.Expression; +import org.apache.camel.builder.Builder; +import org.apache.camel.builder.LanguageBuilderFactory; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * A token is literal text unless regex is true (CAMEL-25015) + */ +public class TokenizeLiteralTokenTest extends ContextTestSupport { + + @Override + public boolean isUseRouteBuilder() { + return false; + } + + private static LanguageBuilderFactory lang() { + return new LanguageBuilderFactory(); + } + + private List<String> split(Expression expression, String body) throws Exception { + context.addRoutes(new RouteBuilder() { + @Override + public void configure() { + from("direct:start").split(expression).to("mock:split"); + } + }); + context.start(); + Exchange out = template.send("direct:start", e -> { + e.getMessage().setBody(body); + e.getMessage().setHeader("sep", "|"); + }); + assertThat(out.getException()).isNull(); + MockEndpoint mock = getMockEndpoint("mock:split"); + return mock.getReceivedExchanges().stream().map(e -> e.getMessage().getBody(String.class)).toList(); + } + + @Test + public void testPipe() throws Exception { + assertThat(split(lang().tokenize().token("|").end(), "a|b|c")).containsExactly("a", "b", "c"); + } + + @Test + public void testDot() throws Exception { + assertThat(split(lang().tokenize().token(".").end(), "a.b.c")).containsExactly("a", "b", "c"); + } + + @Test + public void testDollar() throws Exception { + assertThat(split(lang().tokenize().token("$").end(), "a$b$c")).containsExactly("a", "b", "c"); + } + + @Test + public void testStar() throws Exception { + assertThat(split(lang().tokenize().token("*").end(), "a*b*c")).containsExactly("a", "b", "c"); + } + + @Test + public void testPipeFromValueBuilder() throws Exception { + assertThat(split(Builder.body().tokenize("|"), "a|b|c")).containsExactly("a", "b", "c"); + } + + @Test + public void testPipeGroupKeepsTokenAsDelimiter() throws Exception { + assertThat(split(lang().tokenize().token("|").group(2).end(), "a|b|c|d")).containsExactly("a|b", "c|d"); + } + + @Test + public void testPipeFromSimpleGroupKeepsTokenAsDelimiter() throws Exception { + assertThat(split(lang().tokenize().token("${header.sep}").group(2).end(), "a|b|c|d")).containsExactly("a|b", "c|d"); + } + + @Test + public void testRegexCharactersWithoutRegexAreLiteral() throws Exception { + assertThat(split(lang().tokenize().token("\\s+").end(), "a b\\s+c")).containsExactly("a b", "c"); + } + + @Test + public void testRegex() throws Exception { + assertThat(split(lang().tokenize().token("\\s+").regex(true).end(), "a b\tc")).containsExactly("a", "b", "c"); + } + + @Test + public void testEscapedPipeWithRegex() throws Exception { + assertThat(split(lang().tokenize().token("\\|").regex(true).end(), "a|b|c")).containsExactly("a", "b", "c"); + } + + @Test + public void testEscapedCarriageReturnNewLine() throws Exception { + // as the token is written in XML or YAML: backslash r backslash n (4 characters) + assertThat(split(lang().tokenize().token("\\r\\n").end(), "a\r\nb\r\nc")).containsExactly("a", "b", "c"); + } + + @Test + public void testEscapedTab() throws Exception { + assertThat(split(lang().tokenize().token("\\t").end(), "a\tb\tc")).containsExactly("a", "b", "c"); + } + + @Test + public void testPairWithParentheses() throws Exception { + assertThat(split(lang().tokenize().token("(").endToken(")").end(), "(a)(b)")).containsExactly("a", "b"); + } + + @Test + public void testPairWithDotInEndToken() throws Exception { + assertThat(split(lang().tokenize().token("[rec]").endToken("END.").end(), "[rec]1 ENDS HERE END.[rec]2END.")) + .containsExactly("1 ENDS HERE ", "2"); + } + + @Test + public void testPairWithBrackets() throws Exception { + assertThat(split(lang().tokenize().token("[").endToken("]").end(), "[a][b]")).containsExactly("a", "b"); + } +} diff --git a/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java b/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java index d3e8a28ac0cf..8e7f105df9c6 100644 --- a/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java +++ b/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java @@ -68,7 +68,7 @@ public final class GroupTokenIterator implements Iterator<Object>, Closeable { // if the iterator is a scanner then it may have a dynamic delimiter // so we need to use the actual evaluated delimiter as token if (LanguageSupport.hasSimpleFunction(token) && it instanceof Scanner scanner) { - this.token = scanner.getDelim(); + this.token = unquote(scanner.getDelim()); } else { this.token = token; } @@ -165,4 +165,16 @@ public final class GroupTokenIterator implements Iterator<Object>, Closeable { public void remove() { it.remove(); } + + /** + * Turns a delimiter quoted with {@link java.util.regex.Pattern#quote(String)} back into its literal text, so the + * parts are joined with the token and not with the quoted pattern. Any other delimiter is returned as is. + */ + private static String unquote(String delim) { + if (delim != null && delim.length() >= 4 && delim.startsWith("\\Q") && delim.endsWith("\\E")) { + // Pattern.quote writes a \E in the text as \E\\E\Q + return delim.substring(2, delim.length() - 2).replace("\\E\\\\E\\Q", "\\E"); + } + return delim; + } } diff --git a/core/camel-support/src/main/java/org/apache/camel/support/builder/ExpressionBuilder.java b/core/camel-support/src/main/java/org/apache/camel/support/builder/ExpressionBuilder.java index bddd202ada29..2a9ec43d0f37 100644 --- a/core/camel-support/src/main/java/org/apache/camel/support/builder/ExpressionBuilder.java +++ b/core/camel-support/src/main/java/org/apache/camel/support/builder/ExpressionBuilder.java @@ -1850,7 +1850,8 @@ public class ExpressionBuilder { } /** - * Returns a tokenize expression which will tokenize the string with the given token + * Returns a tokenize expression which will tokenize the string with the given token. The token is matched as + * literal text, use {@link #regexTokenizeExpression(Expression, String)} for a regular expression. */ public static Expression tokenizeExpression( final Expression expression, @@ -1859,7 +1860,8 @@ public class ExpressionBuilder { } /** - * Returns a tokenize expression which will tokenize the string with the given token + * Returns a tokenize expression which will tokenize the string with the given token. The token is matched as + * literal text, use {@link #regexTokenizeExpression(Expression, String)} for a regular expression. */ public static Expression tokenizeExpression( final Expression expression, @@ -1869,7 +1871,8 @@ public class ExpressionBuilder { public Object evaluate(Exchange exchange) { String text = token.evaluate(exchange, String.class); Object value = expression.evaluate(exchange, Object.class); - return ExchangeHelper.getScanner(exchange, value, text); + // the scanner is regular expression based, so quote the token to match it as literal text + return ExchangeHelper.getScanner(exchange, value, text != null ? Pattern.quote(text) : null); } @Override diff --git a/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenPairExpressionIterator.java b/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenPairExpressionIterator.java index dcf4dfa98e8e..f6b73132c617 100644 --- a/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenPairExpressionIterator.java +++ b/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenPairExpressionIterator.java @@ -20,6 +20,7 @@ import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; +import java.util.regex.Pattern; import org.apache.camel.CamelContext; import org.apache.camel.Exchange; @@ -166,9 +167,8 @@ public class TokenPairExpressionIterator extends ExpressionAdapter { static class TokenPairIterator implements Iterator<Object>, Closeable { final String startToken; - String scanStartToken; final String endToken; - String scanEndToken; + final String scanEndToken; final boolean includeTokens; final InputStream in; final String charset; @@ -182,22 +182,8 @@ public class TokenPairExpressionIterator extends ExpressionAdapter { this.in = in; this.charset = charset; - // make sure [ and ] is escaped as we use scanner which is reg exp based - // where [ and ] have special meaning - scanStartToken = startToken; - if (scanStartToken.startsWith("[")) { - scanStartToken = "\\" + scanStartToken; - } - if (scanStartToken.endsWith("]")) { - scanStartToken = scanStartToken.substring(0, startToken.length() - 1) + "\\]"; - } - scanEndToken = endToken; - if (scanEndToken.startsWith("[")) { - scanEndToken = "\\" + scanEndToken; - } - if (scanEndToken.endsWith("]")) { - scanEndToken = scanEndToken.substring(0, scanEndToken.length() - 1) + "\\]"; - } + // the scanner is regular expression based, so quote the end token to match it as literal text + scanEndToken = Pattern.quote(endToken); } void init() { diff --git a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc index b50ad720d318..5afdb6fe77a9 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc @@ -366,6 +366,23 @@ now receive the input that was split, instead of that one split item. A route th read the `CamelSplitResult` exchange property, set as before: `SplitResult.getFailures()` has the index and the exception of the failed items. +=== camel-core - Tokenize: the token is literal text unless regex is true + +The token of the xref:languages:tokenize-language.adoc[Tokenize] language is now matched as literal text, as the +`regex` option (default `false`) describes. Before, the token was always compiled as a regular expression, so a +delimiter such as `|`, `.`, `$` or `*` split every character, did not split, or failed with a `PatternSyntaxException`. +The end token of the pair mode (`endToken`) is also literal text now. + +A route that relies on a regular expression as token, such as `\s+` or `(\W+)\s*`, must now set `regex=true`: + +[source,java] +---- +.split().tokenize("\\s+", true) +---- + +In the XML and YAML DSL, a token with the escapes `\n`, `\r` or `\t` (for example `\r\n`) is turned into the new line, +carriage return and tab characters, so it works as before. Before, only a leading `\n` was turned into a new line. + === camel-main - a failure to start is logged A Camel Main application that fails to start now logs why, at ERROR: `Error starting Camel: ...` with the stack trace,
