This is an automated email from the ASF dual-hosted git repository.
zihaoxiang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new dddf2dc834 [Fix-16914] [Task] ParameterUtils will throw exception when
the input contains '$[xx]' (#16916)
dddf2dc834 is described below
commit dddf2dc83428e4bb5eb7e1286a5b5f53d5dbaf8b
Author: Wenjun Ruan <[email protected]>
AuthorDate: Tue Dec 24 17:05:38 2024 +0800
[Fix-16914] [Task] ParameterUtils will throw exception when the input
contains '$[xx]' (#16916)
---
.../task/api/parser/PropertyPlaceholderHelper.java | 2 +-
.../task/api/parser/TimePlaceholderUtils.java | 146 ++++++---------------
.../plugin/task/api/utils/ParameterUtils.java | 21 +--
.../task/api/parser/TimePlaceholderUtilsTest.java | 9 +-
.../plugin/task/api/utils/ParameterUtilsTest.java | 47 +++----
5 files changed, 78 insertions(+), 147 deletions(-)
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java
index e10657d084..dbf484fc05 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java
@@ -119,7 +119,7 @@ public class PropertyPlaceholderHelper {
*/
public String replacePlaceholders(String value, PlaceholderResolver
placeholderResolver) {
notNull(value, "'value' must not be null");
- return parseStringValue(value, placeholderResolver, new
HashSet<String>());
+ return parseStringValue(value, placeholderResolver, new HashSet<>());
}
protected String parseStringValue(
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java
index 42d95083f1..24bb791870 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java
@@ -71,44 +71,6 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class TimePlaceholderUtils {
- /**
- * Prefix of the position to be replaced
- */
- public static final String PLACEHOLDER_PREFIX = "$[";
-
- /**
- * The suffix of the position to be replaced
- */
- public static final String PLACEHOLDER_SUFFIX = "]";
-
- /**
- * Replaces all placeholders of format {@code ${name}} with the value
returned
- * from the supplied {@link PropertyPlaceholderHelper.PlaceholderResolver}.
- *
- * @param value the value containing the
placeholders to be replaced
- * @param date custom date
- * @param ignoreUnresolvablePlaceholders ignore unresolvable placeholders
- * @return the supplied value with placeholders replaced inline
- */
- public static String replacePlaceholders(String value, Date date, boolean
ignoreUnresolvablePlaceholders) {
- PropertyPlaceholderHelper strictHelper =
getPropertyPlaceholderHelper(false);
- PropertyPlaceholderHelper nonStrictHelper =
getPropertyPlaceholderHelper(true);
-
- PropertyPlaceholderHelper helper = (ignoreUnresolvablePlaceholders ?
nonStrictHelper : strictHelper);
- return helper.replacePlaceholders(value, new
TimePlaceholderResolver(value, date));
- }
-
- /**
- * Creates a new {@code PropertyPlaceholderHelper} that uses the supplied
prefix and suffix.
- *
- * @param ignoreUnresolvablePlaceholders indicates whether unresolvable
placeholders should
- * be ignored ({@code true}) or
cause an exception ({@code false})
- */
- private static PropertyPlaceholderHelper
getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) {
- return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX,
PLACEHOLDER_SUFFIX, null,
- ignoreUnresolvablePlaceholders);
- }
-
/**
* calculate expression's value
*
@@ -303,79 +265,35 @@ public class TimePlaceholderUtils {
}
/**
- * Placeholder replacement resolver
- */
- private static class TimePlaceholderResolver
- implements
- PropertyPlaceholderHelper.PlaceholderResolver {
-
- private final String value;
-
- private final Date date;
-
- public TimePlaceholderResolver(String value, Date date) {
- this.value = value;
- this.date = date;
- }
-
- @Override
- public String resolvePlaceholder(String placeholderName) {
- try {
- return calculateTime(placeholderName, date);
- } catch (Exception ex) {
- log.error("resolve placeholder '{}' in [ {} ]",
placeholderName, value, ex);
- return null;
- }
- }
- }
-
- /**
- * return the formatted date according to the corresponding date format
- *
- * @param expression date expression
- * @param date date
- * @return reformat date
- */
- public static String getPlaceHolderTime(String expression, Date date) {
- if (StringUtils.isBlank(expression)) {
- throw new IllegalArgumentException("expression is null");
- }
- if (null == date) {
- throw new IllegalArgumentException("date is null");
- }
- return calculateTime(expression, date);
- }
-
- /**
- * calculate time
+ * Format time expression with given date, the date cannot be null.
+ * <p> If the expression is not a time expression, return the original
expression.
*
- * @param date date
- * @return calculate time
*/
- private static String calculateTime(String expression, Date date) {
+ public static String formatTimeExpression(final String timeExpression,
final Date date,
+ final boolean
ignoreInvalidExpression) {
// After N years: $[add_months(yyyyMMdd,12*N)], the first N months:
$[add_months(yyyyMMdd,-N)], etc
if (date == null) {
- throw new IllegalArgumentException("Cannot parse the expression: "
+ expression + ", date is null");
+ throw new IllegalArgumentException("Cannot parse the expression: "
+ timeExpression + ", date is null");
+ }
+ if (StringUtils.isEmpty(timeExpression)) {
+ if (ignoreInvalidExpression) {
+ return timeExpression;
+ }
+ throw new IllegalArgumentException("Cannot format the date" + date
+ " with null timeExpression");
}
try {
- if (expression.startsWith(TIMESTAMP)) {
- String timeExpression =
expression.substring(TIMESTAMP.length() + 1, expression.length() - 1);
-
- Map.Entry<Date, String> entry =
calcTimeExpression(timeExpression, date);
-
- String dateStr = DateUtils.format(entry.getKey(),
entry.getValue());
-
- Date timestamp = DateUtils.parse(dateStr,
PARAMETER_FORMAT_TIME);
-
- return String.valueOf(timestamp.getTime() / 1000);
+ if (timeExpression.startsWith(TIMESTAMP)) {
+ return calculateTimeStamp(timeExpression, date);
}
- if (expression.startsWith(YEAR_WEEK)) {
- return calculateYearWeek(expression, date);
+ if (timeExpression.startsWith(YEAR_WEEK)) {
+ return calculateYearWeek(timeExpression, date);
}
- Map.Entry<Date, String> entry = calcTimeExpression(expression,
date);
- return DateUtils.format(entry.getKey(), entry.getValue());
+ return calcTimeExpression(timeExpression, date);
} catch (Exception e) {
- throw new IllegalArgumentException("Unsupported placeholder
expression: " + expression, e);
+ if (ignoreInvalidExpression) {
+ return timeExpression;
+ }
+ throw new IllegalArgumentException("Unsupported placeholder
expression: " + timeExpression, e);
}
}
@@ -385,7 +303,7 @@ public class TimePlaceholderUtils {
* @param date date
* @return week of year
*/
- public static String calculateYearWeek(String expression, Date date) {
+ private static String calculateYearWeek(final String expression, final
Date date) {
String dataFormat = expression.substring(YEAR_WEEK.length() + 1,
expression.length() - 1);
@@ -465,6 +383,19 @@ public class TimePlaceholderUtils {
return weekYearStr;
}
+ private static String calculateTimeStamp(final String expression, final
Date date) {
+ final String timeExpression = expression.substring(TIMESTAMP.length()
+ 1, expression.length() - 1);
+ final String calculatedTimeExpression =
calcTimeExpression(timeExpression, date);
+ if (StringUtils.equals(expression, calculatedTimeExpression)) {
+ return expression;
+ }
+ final Date timestamp = DateUtils.parse(calculatedTimeExpression,
PARAMETER_FORMAT_TIME);
+ if (timestamp == null) {
+ throw new IllegalArgumentException("Cannot parse the expression: "
+ expression + " with date: " + date);
+ }
+ return String.valueOf(timestamp.getTime() / 1000);
+ }
+
/**
* calculate time expresstion
*
@@ -472,8 +403,8 @@ public class TimePlaceholderUtils {
* @param date date
* @return map with date, date format
*/
- public static Map.Entry<Date, String> calcTimeExpression(String
expression, Date date) {
- Map.Entry<Date, String> resultEntry;
+ private static String calcTimeExpression(final String expression, final
Date date) {
+ final Map.Entry<Date, String> resultEntry;
if (expression.startsWith(ADD_MONTHS)) {
resultEntry = calcMonths(expression, date);
@@ -500,8 +431,9 @@ public class TimePlaceholderUtils {
} else {
resultEntry = calcMinutes(expression, date);
}
-
- return resultEntry;
+ final Date finalDate = resultEntry.getKey();
+ final String finalFormat = resultEntry.getValue();
+ return DateUtils.format(finalDate, finalFormat);
}
/**
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java
index 9733453b2d..9a7b9a86eb 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java
@@ -49,8 +49,6 @@ public class ParameterUtils {
private static final Pattern DATE_PARSE_PATTERN =
Pattern.compile("\\$\\[([^\\$\\]]+)]");
- private static final Pattern DATE_START_PATTERN =
Pattern.compile("^[0-9]");
-
private static final char PARAM_REPLACE_CHAR = '?';
private ParameterUtils() {
@@ -288,21 +286,24 @@ public class ParameterUtils {
return map;
}
- private static String dateTemplateParse(String templateStr, Date date) {
- if (templateStr == null) {
- return null;
+ // Replace the time placeholder in the template string with the given
actual time value
+ // If the templateStr does not contain a time placeholder, will return the
original string
+ private static String dateTemplateParse(final String templateStr, final
Date date) {
+ if (StringUtils.isEmpty(templateStr)) {
+ return templateStr;
}
- StringBuffer newValue = new StringBuffer(templateStr.length());
+ final StringBuffer newValue = new StringBuffer(templateStr.length());
- Matcher matcher = DATE_PARSE_PATTERN.matcher(templateStr);
+ final Matcher matcher = DATE_PARSE_PATTERN.matcher(templateStr);
while (matcher.find()) {
- String key = matcher.group(1);
- if (DATE_START_PATTERN.matcher(key).matches()) {
+ final String key = matcher.group(1);
+ String value = TimePlaceholderUtils.formatTimeExpression(key,
date, true);
+ if (StringUtils.equals(key, value)) {
+ // There is no corresponding time format, so the original
string is retained
continue;
}
- String value = TimePlaceholderUtils.getPlaceHolderTime(key, date);
matcher.appendReplacement(newValue, value);
}
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtilsTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtilsTest.java
index b1b992e857..86d69a03c7 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtilsTest.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtilsTest.java
@@ -27,7 +27,6 @@ import
org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import java.util.Date;
import java.util.Map;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class TimePlaceholderUtilsTest {
@@ -131,10 +130,8 @@ public class TimePlaceholderUtilsTest {
}
@Test
- void getPlaceHolderTime() {
- IllegalArgumentException illegalArgumentException =
Assertions.assertThrows(IllegalArgumentException.class,
- () ->
TimePlaceholderUtils.getPlaceHolderTime("$[week_last_day(yyyy-MM-dd,0) - 1]",
new Date()));
- assertEquals("Unsupported placeholder expression:
$[week_last_day(yyyy-MM-dd,0) - 1]",
- illegalArgumentException.getMessage());
+ void formatTimeExpressionWithInvalidExpression() {
+ assertEquals("$[week_last_day(yyyy-MM-dd,0) - 1]",
+
TimePlaceholderUtils.formatTimeExpression("$[week_last_day(yyyy-MM-dd,0) - 1]",
new Date(), true));
}
}
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java
index 1c0dbc7c06..6a25c573a7 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java
@@ -17,18 +17,16 @@
package org.apache.dolphinscheduler.plugin.task.api.utils;
-import static
org.apache.dolphinscheduler.plugin.task.api.parser.TimePlaceholderUtils.replacePlaceholders;
+import static
org.apache.dolphinscheduler.plugin.task.api.parser.PlaceholderUtils.replacePlaceholders;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import org.apache.dolphinscheduler.common.constants.DateConstants;
-import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
-import org.apache.dolphinscheduler.plugin.task.api.parser.PlaceholderUtils;
import java.text.ParseException;
-import java.util.Date;
import java.util.HashMap;
import java.util.Map;
@@ -51,8 +49,8 @@ public class ParameterUtilsTest {
JSONUtils.toJsonString(Lists.newArrayList(3.1415, 2.44,
3.44))));
String sql = ParameterUtils.expandListParameter(params,
"select * from test where col1 in ('${col1}') and
date='${date}' and col2 in ('${col2}')");
- Assertions.assertEquals("select * from test where col1 in (?,?,?) and
date=? and col2 in (?,?,?)", sql);
- Assertions.assertEquals(7, params.size());
+ assertEquals("select * from test where col1 in (?,?,?) and date=? and
col2 in (?,?,?)", sql);
+ assertEquals(7, params.size());
Map<Integer, Property> params2 = new HashMap<>();
params2.put(1,
@@ -60,8 +58,8 @@ public class ParameterUtilsTest {
params2.put(2, new Property("date", Direct.IN, DataType.DATE,
"2020-06-30"));
String sql2 = ParameterUtils.expandListParameter(params2,
"select * from test where col1 in ('${col}') and
date='${date}'");
- Assertions.assertEquals("select * from test where col1 in (?) and
date=?", sql2);
- Assertions.assertEquals(2, params2.size());
+ assertEquals("select * from test where col1 in (?) and date=?", sql2);
+ assertEquals(2, params2.size());
}
@@ -77,21 +75,14 @@ public class ParameterUtilsTest {
// parameterStringăparameterMap is not null
String parameterString = "test_parameter";
- Assertions.assertEquals(parameterString,
+ assertEquals(parameterString,
ParameterUtils.convertParameterPlaceholders(parameterString,
parameterMap));
// replace variable ${} form
parameterMap.put("testParameter2", "${testParameter}");
- Assertions.assertEquals(parameterString,
- PlaceholderUtils.replacePlaceholders(parameterString,
parameterMap, true));
+ assertEquals(parameterString,
+ replacePlaceholders(parameterString, parameterMap, true));
- // replace time $[...] form, eg. $[yyyyMMdd]
- Date cronTime = new Date();
- Assertions.assertEquals(parameterString,
replacePlaceholders(parameterString, cronTime, true));
-
- // replace time $[...] form, eg. $[yyyyMMdd]
- Date cronTimeStr = DateUtils.stringToDate("2019-02-02 00:00:00");
- Assertions.assertEquals(parameterString,
replacePlaceholders(parameterString, cronTimeStr, true));
}
@Test
@@ -103,20 +94,30 @@ public class ParameterUtilsTest {
parameterMap.put("user", "Kris");
parameterMap.put(DateConstants.PARAMETER_DATETIME, "20201201123000");
parameterString =
ParameterUtils.convertParameterPlaceholders(parameterString, parameterMap);
- Assertions.assertEquals(
+ assertEquals(
"Kris is userName, '$[1]' '20221201' '20181201' '20210301'
'20200801' '20201215' '20201117' '20201204' '$[0]' '20201128' '143000'
'113000' '123300' '122800' '$[3]'",
parameterString);
}
+ @Test
+ public void convertParameterPlaceholdersWithPunct() {
+ final String sql = "get_json_object(json_array, '$[*].pageId')";
+ final Map<String, String> paramsMap = new HashMap<>();
+ paramsMap.put("key", "value");
+
+ assertEquals("get_json_object(json_array, '$[*].pageId')",
+ ParameterUtils.convertParameterPlaceholders(sql, paramsMap));
+ }
+
/**
* Test handleEscapes
*/
@Test
- public void testHandleEscapes() throws Exception {
+ public void testHandleEscapes() {
Assertions.assertNull(ParameterUtils.handleEscapes(null));
- Assertions.assertEquals("", ParameterUtils.handleEscapes(""));
- Assertions.assertEquals("test Parameter",
ParameterUtils.handleEscapes("test Parameter"));
- Assertions.assertEquals("////%test////%Parameter",
ParameterUtils.handleEscapes("%test%Parameter"));
+ assertEquals("", ParameterUtils.handleEscapes(""));
+ assertEquals("test Parameter", ParameterUtils.handleEscapes("test
Parameter"));
+ assertEquals("////%test////%Parameter",
ParameterUtils.handleEscapes("%test%Parameter"));
}
}
