Repository: nifi Updated Branches: refs/heads/master d126743d9 -> 72a5e6bfb
NIFI-4407: Updated Expression Language Guide to provide details about how escaping $ is accomplished, and when the $ character should and should not be escaped. Fixed bug in the Query compiler that mistakenly would blindly replace 484 with $ even when the 484 did not precede an Expression. Added additional test cases. Signed-off-by: Pierre Villard <pierre.villard...@gmail.com> This closes #2898. Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/72a5e6bf Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/72a5e6bf Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/72a5e6bf Branch: refs/heads/master Commit: 72a5e6bfba1597f6174223f3afc85de478d49898 Parents: d126743 Author: Mark Payne <marka...@hotmail.com> Authored: Mon Jul 16 13:27:46 2018 -0400 Committer: Pierre Villard <pierre.villard...@gmail.com> Committed: Wed Jul 18 10:52:13 2018 +0200 ---------------------------------------------------------------------- .../attribute/expression/language/Query.java | 74 ++++-- .../language/compile/ExpressionCompiler.java | 230 +++++++++++-------- .../expression/language/TestQuery.java | 83 ++++--- .../asciidoc/expression-language-guide.adoc | 44 +++- 4 files changed, 279 insertions(+), 152 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/72a5e6bf/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java index 8f549a5..e701341 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java @@ -16,20 +16,21 @@ */ package org.apache.nifi.attribute.expression.language; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - import org.antlr.runtime.tree.Tree; import org.apache.nifi.attribute.expression.language.compile.ExpressionCompiler; import org.apache.nifi.attribute.expression.language.evaluation.Evaluator; import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; +import org.apache.nifi.attribute.expression.language.evaluation.selection.AttributeEvaluator; import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException; import org.apache.nifi.expression.AttributeExpression.ResultType; import org.apache.nifi.expression.AttributeValueDecorator; import org.apache.nifi.processor.exception.ProcessException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + /** * Class used for creating and evaluating NiFi Expression Language. Once a Query * has been created, it may be evaluated using the evaluate methods exactly @@ -206,8 +207,7 @@ public class Query { } final String value = evaluated.toString(); - final String escaped = value.replace("$$", "$"); - return decorator == null ? escaped : decorator.decorate(escaped); + return decorator == null ? value : decorator.decorate(value); } static String evaluateExpressions(final String rawValue, Map<String, String> expressionMap, final AttributeValueDecorator decorator, final Map<String, String> stateVariables) @@ -239,6 +239,42 @@ public class Query { return new Query(text, tree, compiler.buildEvaluator(tree)); } + private static String unescapeLeadingDollarSigns(final String value) { + final int index = value.indexOf("{"); + if (index < 0) { + return value.replace("$$", "$"); + } else { + final String prefix = value.substring(0, index); + return prefix.replace("$$", "$") + value.substring(index); + } + } + + private static String unescapeTrailingDollarSigns(final String value, final boolean escapeIfAllDollars) { + if (!value.endsWith("$")) { + return value; + } + + // count number of $$ at end of string + int dollars = 0; + for (int i=value.length()-1; i >= 0; i--) { + final char c = value.charAt(i); + if (c == '$') { + dollars++; + } else { + break; + } + } + + // If the given argument consists solely of $ characters, then we + if (dollars == value.length() && !escapeIfAllDollars) { + return value; + } + + final int charsToRemove = dollars / 2; + final int newLength = value.length() - charsToRemove; + return value.substring(0, newLength); + } + public static PreparedQuery prepare(final String query) throws AttributeExpressionLanguageParsingException { if (query == null) { @@ -248,7 +284,11 @@ public class Query { final List<Range> ranges = extractExpressionRanges(query); if (ranges.isEmpty()) { - return new EmptyPreparedQuery(query.replace("$$", "$")); + // While in the other cases below, we are simply replacing "$$" with "$", we have to do this + // a bit differently. We want to treat $$ as an escaped $ only if it immediately precedes the + // start of an Expression, which is the case below. Here, we did not detect the start of an Expression + // and as such as must use the #unescape method instead of a simple replace() function. + return new EmptyPreparedQuery(unescape(query)); } final ExpressionCompiler compiler = new ExpressionCompiler(); @@ -258,14 +298,22 @@ public class Query { int lastIndex = 0; for (final Range range : ranges) { + final String treeText = unescapeLeadingDollarSigns(query.substring(range.getStart(), range.getEnd() + 1)); + final CompiledExpression compiledExpression = compiler.compile(treeText); + if (range.getStart() > lastIndex) { - final String substring = query.substring(lastIndex, range.getStart()).replace("$$", "$"); + String substring = unescapeLeadingDollarSigns(query.substring(lastIndex, range.getStart())); + + // If this string literal evaluator immediately precedes an Attribute Reference, then we need to consider the String Literal to be + // Escaping if it ends with $$'s, otherwise not. We also want to avoid un-escaping if the expression consists solely of $$, because + // those would have been addressed by the previous #unescapeLeadingDollarSigns() call already. + if (compiledExpression.getRootEvaluator() instanceof AttributeEvaluator) { + substring = unescapeTrailingDollarSigns(substring, false); + } + expressions.add(new StringLiteralExpression(substring)); - lastIndex = range.getEnd() + 1; } - final String treeText = query.substring(range.getStart(), range.getEnd() + 1).replace("$$", "$"); - final CompiledExpression compiledExpression = compiler.compile(treeText); expressions.add(compiledExpression); lastIndex = range.getEnd() + 1; @@ -273,7 +321,7 @@ public class Query { final Range lastRange = ranges.get(ranges.size() - 1); if (lastRange.getEnd() + 1 < query.length()) { - final String treeText = query.substring(lastRange.getEnd() + 1).replace("$$", "$"); + final String treeText = unescapeLeadingDollarSigns(query.substring(lastRange.getEnd() + 1)); expressions.add(new StringLiteralExpression(treeText)); } http://git-wip-us.apache.org/repos/asf/nifi/blob/72a5e6bf/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/compile/ExpressionCompiler.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/compile/ExpressionCompiler.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/compile/ExpressionCompiler.java index 622becd..4a4a204 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/compile/ExpressionCompiler.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/compile/ExpressionCompiler.java @@ -17,104 +17,6 @@ package org.apache.nifi.attribute.expression.language.compile; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_ATTRIBUTES; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_DELINEATED_VALUES; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_MATCHING_ATTRIBUTES; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.AND; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_ATTRIBUTE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_DELINEATED_VALUE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_MATCHING_ATTRIBUTE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.APPEND; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ATTRIBUTE_REFERENCE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ATTR_NAME; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.BASE64_DECODE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.BASE64_ENCODE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.CONTAINS; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.COUNT; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.DECIMAL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.DIVIDE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ENDS_WITH; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EQUALS; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EQUALS_IGNORE_CASE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ESCAPE_CSV; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ESCAPE_HTML3; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ESCAPE_HTML4; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ESCAPE_JSON; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ESCAPE_XML; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EXPRESSION; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FALSE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FIND; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FORMAT; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FROM_RADIX; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GET_DELIMITED_FIELD; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GET_STATE_VALUE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GREATER_THAN; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GREATER_THAN_OR_EQUAL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.HOSTNAME; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IF_ELSE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IN; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.INDEX_OF; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IP; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IS_EMPTY; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IS_NULL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.JOIN; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.JSON_PATH; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LAST_INDEX_OF; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LENGTH; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LESS_THAN; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LESS_THAN_OR_EQUAL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MATCHES; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MATH; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MINUS; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MOD; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MULTIPLY; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MULTI_ATTRIBUTE_REFERENCE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NEXT_INT; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOT; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOT_NULL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOW; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.OR; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.PLUS; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.PREPEND; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.RANDOM; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_ALL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_EMPTY; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_FIRST; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_NULL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STARTS_WITH; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STRING_LITERAL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_AFTER; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_AFTER_LAST; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_BEFORE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_BEFORE_LAST; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_DATE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_DECIMAL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_LITERAL; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_LOWER; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_NUMBER; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_RADIX; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_STRING; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_UPPER; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TRIM; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TRUE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UNESCAPE_CSV; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UNESCAPE_HTML3; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UNESCAPE_HTML4; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UNESCAPE_JSON; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UNESCAPE_XML; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.URL_DECODE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.URL_ENCODE; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UUID; -import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.WHOLE_NUMBER; - -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - import org.antlr.runtime.ANTLRStringStream; import org.antlr.runtime.CharStream; import org.antlr.runtime.CommonTokenStream; @@ -219,6 +121,104 @@ import org.apache.nifi.attribute.expression.language.exception.AttributeExpressi import org.apache.nifi.expression.AttributeExpression.ResultType; import org.apache.nifi.flowfile.FlowFile; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_ATTRIBUTES; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_DELINEATED_VALUES; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_MATCHING_ATTRIBUTES; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.AND; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_ATTRIBUTE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_DELINEATED_VALUE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ANY_MATCHING_ATTRIBUTE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.APPEND; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ATTRIBUTE_REFERENCE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ATTR_NAME; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.BASE64_DECODE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.BASE64_ENCODE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.CONTAINS; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.COUNT; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.DECIMAL; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.DIVIDE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ENDS_WITH; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EQUALS; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EQUALS_IGNORE_CASE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ESCAPE_CSV; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ESCAPE_HTML3; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ESCAPE_HTML4; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ESCAPE_JSON; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ESCAPE_XML; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EXPRESSION; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FALSE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FIND; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FORMAT; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FROM_RADIX; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GET_DELIMITED_FIELD; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GET_STATE_VALUE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GREATER_THAN; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.GREATER_THAN_OR_EQUAL; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.HOSTNAME; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IF_ELSE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IN; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.INDEX_OF; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IP; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IS_EMPTY; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.IS_NULL; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.JOIN; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.JSON_PATH; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LAST_INDEX_OF; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LENGTH; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LESS_THAN; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LESS_THAN_OR_EQUAL; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MATCHES; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MATH; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MINUS; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MOD; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MULTIPLY; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MULTI_ATTRIBUTE_REFERENCE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NEXT_INT; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOT; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOT_NULL; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.NOW; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.OR; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.PLUS; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.PREPEND; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.RANDOM; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_ALL; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_EMPTY; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_FIRST; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.REPLACE_NULL; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STARTS_WITH; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.STRING_LITERAL; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_AFTER; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_AFTER_LAST; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_BEFORE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.SUBSTRING_BEFORE_LAST; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_DATE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_DECIMAL; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_LITERAL; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_LOWER; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_NUMBER; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_RADIX; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_STRING; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TO_UPPER; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TRIM; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.TRUE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UNESCAPE_CSV; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UNESCAPE_HTML3; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UNESCAPE_HTML4; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UNESCAPE_JSON; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UNESCAPE_XML; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.URL_DECODE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.URL_ENCODE; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UUID; +import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.WHOLE_NUMBER; + public class ExpressionCompiler { private final Set<Evaluator<?>> evaluators = new HashSet<>(); @@ -1066,6 +1066,26 @@ public class ExpressionCompiler { return evaluator; } + private String unescapeTrailingDollarSigns(final String value) { + if (!value.endsWith("$")) { + return value; + } + + // count number of $$ at end of string + int dollars = 0; + for (int i=value.length()-1; i >= 0; i--) { + final char c = value.charAt(i); + if (c == '$') { + dollars++; + } else { + break; + } + } + + final int charsToRemove = dollars / 2; + final int newLength = value.length() - charsToRemove; + return value.substring(0, newLength); + } private Evaluator<String> newStringLiteralEvaluator(final String literalValue) { if (literalValue == null || literalValue.length() < 2) { @@ -1081,12 +1101,20 @@ public class ExpressionCompiler { int lastIndex = 0; for (final Range range : ranges) { + final String treeText = literalValue.substring(range.getStart(), range.getEnd() + 1); + final Evaluator<?> evaluator = buildEvaluator(compileTree(treeText)); + if (range.getStart() > lastIndex) { - evaluators.add(newStringLiteralEvaluator(literalValue.substring(lastIndex, range.getStart()))); + // If this string literal evaluator immediately precedes an Attribute Reference, then we need to consider the String Literal to be + // Escaping if it ends with $$'s, otherwise not. + if (evaluator instanceof AttributeEvaluator) { + evaluators.add(newStringLiteralEvaluator(unescapeTrailingDollarSigns(literalValue.substring(lastIndex, range.getStart())))); + } else { + evaluators.add(newStringLiteralEvaluator(literalValue.substring(lastIndex, range.getStart()))); + } } - final String treeText = literalValue.substring(range.getStart(), range.getEnd() + 1); - evaluators.add(buildEvaluator(compileTree(treeText))); + evaluators.add(evaluator); lastIndex = range.getEnd() + 1; } http://git-wip-us.apache.org/repos/asf/nifi/blob/72a5e6bf/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java index 6275bac..7fd8747 100644 --- a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java +++ b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java @@ -16,15 +16,19 @@ */ package org.apache.nifi.attribute.expression.language; -import static java.lang.Double.NEGATIVE_INFINITY; -import static java.lang.Double.NaN; -import static java.lang.Double.POSITIVE_INFINITY; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.greaterThan; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import org.antlr.runtime.tree.Tree; +import org.apache.nifi.attribute.expression.language.Query.Range; +import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult; +import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; +import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageException; +import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException; +import org.apache.nifi.expression.AttributeExpression.ResultType; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.registry.VariableRegistry; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; +import org.mockito.Mockito; import java.io.BufferedInputStream; import java.io.IOException; @@ -40,19 +44,15 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import org.antlr.runtime.tree.Tree; -import org.apache.nifi.attribute.expression.language.Query.Range; -import org.apache.nifi.attribute.expression.language.evaluation.NumberQueryResult; -import org.apache.nifi.attribute.expression.language.evaluation.QueryResult; -import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageException; -import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageParsingException; -import org.apache.nifi.expression.AttributeExpression.ResultType; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.registry.VariableRegistry; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; -import org.mockito.Mockito; +import static java.lang.Double.NEGATIVE_INFINITY; +import static java.lang.Double.NaN; +import static java.lang.Double.POSITIVE_INFINITY; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class TestQuery { @@ -78,6 +78,8 @@ public class TestQuery { public void testPrepareWithEscapeChar() { final Map<String, String> variables = Collections.singletonMap("foo", "bar"); + assertEquals("bar${foo}$bar", Query.prepare("${foo}$${foo}$$${foo}").evaluateExpressions(variables, null)); + final PreparedQuery onlyEscapedQuery = Query.prepare("$${foo}"); final String onlyEscapedEvaluated = onlyEscapedQuery.evaluateExpressions(variables, null); assertEquals("${foo}", onlyEscapedEvaluated); @@ -85,8 +87,6 @@ public class TestQuery { final PreparedQuery mixedQuery = Query.prepare("${foo}$${foo}"); final String mixedEvaluated = mixedQuery.evaluateExpressions(variables, null); assertEquals("bar${foo}", mixedEvaluated); - - assertEquals("bar${foo}$bar", Query.prepare("${foo}$${foo}$$${foo}").evaluateExpressions(variables, null)); } private void assertValid(final String query) { @@ -167,14 +167,33 @@ public class TestQuery { @Test public void testEscape() { final Map<String, String> attributes = new HashMap<>(); - attributes.put("attr", "My Value"); - attributes.put("${xx}", "hello"); - - assertEquals("My Value", evaluateQueryForEscape("${attr}", attributes)); - assertEquals("${attr}", evaluateQueryForEscape("$${attr}", attributes)); - assertEquals("$My Value", evaluateQueryForEscape("$$${attr}", attributes)); - assertEquals("$${attr}", evaluateQueryForEscape("$$$${attr}", attributes)); - assertEquals("$$My Value", evaluateQueryForEscape("$$$$${attr}", attributes)); + attributes.put("abc", "xyz"); + attributes.put("xyz", "hello"); + attributes.put("$xyz", "good-bye"); + attributes.put("${abc}", "good-bye"); + + assertEquals("$$xyz", evaluateQueryForEscape("$$$$${abc}", attributes)); + assertEquals("xyz", evaluateQueryForEscape("${abc}", attributes)); + assertEquals("${abc}", evaluateQueryForEscape("$${abc}", attributes)); + assertEquals("$xyz", evaluateQueryForEscape("$$${abc}", attributes)); + assertEquals("$${abc}", evaluateQueryForEscape("$$$${abc}", attributes)); + + assertEquals( "Unescaped $$${5 because no closing brace", evaluateQueryForEscape("Unescaped $$${5 because no closing brace", attributes)); + assertEquals( "Unescaped $ because no closing brace", evaluateQueryForEscape("Unescaped $$${'5'} because no closing brace", attributes)); + + assertEquals("I owe you $5", evaluateQueryForEscape("I owe you $5", attributes)); + assertEquals("You owe me $$5 too", evaluateQueryForEscape("You owe me $$5 too", attributes)); + assertEquals("Unescaped $$${5 because no closing brace", evaluateQueryForEscape("Unescaped $$${5 because no closing brace", attributes)); + assertEquals("xyz owes me $5", evaluateQueryForEscape("${abc} owes me $5", attributes)); + assertEquals("xyz owes me ${5", evaluateQueryForEscape("${abc} owes me ${5", attributes)); + assertEquals("xyz owes me ", evaluateQueryForEscape("${abc} owes me ${'5'}", attributes)); + assertEquals("xyz owes me $", evaluateQueryForEscape("${abc} owes me $$${'5'}", attributes)); + + assertEquals("SNAP$$$$$$$$$", evaluateQueryForEscape("${literal('SNAP$$$$$$$$$')}", attributes)); + assertEquals("SNAP$$", evaluateQueryForEscape("${literal('SNAP$$')}", attributes)); + assertEquals("hello", evaluateQueryForEscape("${${abc}}", attributes)); + assertEquals("good-bye", evaluateQueryForEscape("${'$$${abc}'}", attributes)); + assertEquals("good-bye", evaluateQueryForEscape("${'$xyz'}", attributes)); } @Test http://git-wip-us.apache.org/repos/asf/nifi/blob/72a5e6bf/nifi-docs/src/main/asciidoc/expression-language-guide.adoc ---------------------------------------------------------------------- diff --git a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc index 56906d8..a58e8f6 100644 --- a/nifi-docs/src/main/asciidoc/expression-language-guide.adoc +++ b/nifi-docs/src/main/asciidoc/expression-language-guide.adoc @@ -142,24 +142,56 @@ a Property supports the Expression Language is determined by the developer of th the Processor is written. However, the application strives to clearly illustrate for each Property whether or not the Expression Language is supported. -In the application, when configuring a Processor property, the User Interface provides an Information +In the application, when configuring a component property, the User Interface provides an Information icon ( image:iconInfo.png["Info"] ) next to the name of the Property. Hovering over this icon with the mouse will provide a tooltip that provides helpful information about the Property. This information includes a description of the Property, the default value (if any), historically configured values (if any), and the evaluation scope of this -property for expression language. There are three values and the evaluation scope of the expression +property for expression language. There are three values and the evaluation scope of the expression language is hierarchical: NONE -> VARIABLE_REGISTRY -> FLOWFILE_ATTRIBUTES. * NONE - expression language is not supported for this property * VARIABLE_REGISTRY is hierarchically constructed as below: -** Variables defined at process group level and then, recursively, up to the higher process group until +** Variables defined at process group level and then, recursively, up to the higher process group until the root process group. -** Variables defined in custom properties files through the nifi.variable.registry.properties property +** Variables defined in custom properties files through the nifi.variable.registry.properties property in nifi.properties file. ** Environment variables defined at JVM level and system properties. -* FLOWFILE_ATTRIBUTES - will use attributes of each individual flow file. - +* FLOWFILE_ATTRIBUTES - will use attributes of each individual flow file, as well as those variables defined +by the Variable Registry, as described above. + +[[escaping]] +=== Escaping Expression Language +There may be times when a property supports Expression Language, but the user wishes to use a literal value +that follows the same syntax as the Expression Language. For example, a user may want to configure a property +value to be the literal text `Hello ${UserName}`. In such a case, this can be accomplished by using an extra +`$` (dollar sign symbol) just before the expression to escape it (i.e., `Hello $${UserName}`). Unless the `$` +character is being used to escape an Expression, it should not be escaped. For example, the value `Hello $$User$$Name` +should not escape the `$` characters, so the literal value that will be used is `Hello $$User$$Name`. + +If more than two `$` characters are encountered sequentially before a `{`, then each pair of `$` characters will +be considered an escaping of the `$` character. The escaping will be performed from left-to-right. +To help illustrate this, consider that the variable `abc` contains the value `xyz`. Then, consider the following +table of Expressions and their corresponding evaluated values: + +.Escaping EL Examples +|======================================================================================== +| Expression | Value | Notes +| `${abc}` | `xyz` | +| `$${abc}` | `${abc}` | +| `$$${abc}` | `$xyz` | +| `$$$${abc}` | `$${abc}` | +| `$$$$${abc}` | `$$xyz` | +| `I owe you $5` | `I owe you $5` | No actual Expression is present here. +| `You owe me $$5 too` | `You owe me $$5 too` | The $ character is not escaped because it does not immediately precede an Expression. +| `Unescaped $$${5 because no closing brace` | `Unescaped $$${5 because no closing brace` | Because there is no closing brace here, there is no actual Expression and hence the $ characters are not +escaped. +| `Unescaped $$${5} because no closing brace` | <Error> | This expression is not valid because it equates to an escaped $, followed by `${5}` and the `${5}` is not a valid Expression. The number +must be escaped. +| `Unescaped $$${'5'} because no closing brace` | `Unescaped $ because no closing brace` | There is no attribute named `5` so the Expression evaluates to an empty string. The `$$` evaluates to a +single (escaped) `$` because it immediately precedes an Expression. +|======================================================================================== [[editor]] === Expression Language Editor
