This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new b4837e4191a branch-4.0: [fix](function) fix date_format fe too many
string input problem #56241 (#56509)
b4837e4191a is described below
commit b4837e4191a782b7f202f497019c839d4fb4eecc
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Sep 27 09:56:04 2025 +0800
branch-4.0: [fix](function) fix date_format fe too many string input
problem #56241 (#56509)
Cherry-picked from #56241
Co-authored-by: dwdwqfwe <[email protected]>
---
.../executable/DateTimeExtractAndTransform.java | 45 ++++-
.../org/apache/doris/nereids/util/DateUtils.java | 199 +++++++++++++++++++++
.../nereids/rules/expression/FoldConstantTest.java | 6 +
.../sql-functions/doc_date_functions_test.out | 3 +
.../sql-functions/doc_date_functions_test.groovy | 32 ++++
5 files changed, 279 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java
index a23826ca2e3..92b8a8d0e9a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java
@@ -43,6 +43,8 @@ import org.apache.doris.nereids.types.DateV2Type;
import org.apache.doris.nereids.types.DecimalV3Type;
import org.apache.doris.nereids.util.DateUtils;
+import org.apache.commons.lang3.StringUtils;
+
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.DayOfWeek;
@@ -65,7 +67,8 @@ import java.util.Locale;
/**
* executable function:
- * year, quarter, month, week, dayOfYear, dayOfweek, dayOfMonth, hour, minute,
second, microsecond
+ * year, quarter, month, week, dayOfYear, dayOfweek, dayOfMonth, hour, minute,
+ * second, microsecond
*/
public class DateTimeExtractAndTransform {
@@ -324,30 +327,60 @@ public class DateTimeExtractAndTransform {
*/
@ExecFunction(name = "date_format")
public static Expression dateFormat(DateLiteral date, StringLikeLiteral
format) {
+ if (StringUtils.trim(format.getValue()).length() > 128) {
+ throw new AnalysisException("The length of format string in
date_format() function should not be greater"
+ + " than 128.");
+ }
+ DateTimeV2Literal datetime = new DateTimeV2Literal(date.getYear(),
date.getMonth(), date.getDay(), 0, 0, 0, 0);
format = (StringLikeLiteral)
SupportJavaDateFormatter.translateJavaFormatter(format);
- return new
VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()).format(
+ return new
VarcharLiteral(DateUtils.dateTimeFormatterChecklength(format.getValue(),
datetime).format(
java.time.LocalDate.of(((int) date.getYear()), ((int)
date.getMonth()), ((int) date.getDay()))));
}
+ /**
+ * datetime arithmetic function date-format
+ */
@ExecFunction(name = "date_format")
public static Expression dateFormat(DateTimeLiteral date,
StringLikeLiteral format) {
+ if (StringUtils.trim(format.getValue()).length() > 128) {
+ throw new AnalysisException("The length of format string in
date_format() function should not be greater"
+ + " than 128.");
+ }
+ DateTimeV2Literal datetime = new DateTimeV2Literal(date.getYear(),
date.getMonth(), date.getDay(),
+ date.getHour(), date.getMinute(), date.getSecond(),
date.getMicroSecond());
format = (StringLikeLiteral)
SupportJavaDateFormatter.translateJavaFormatter(format);
- return new
VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()).format(
+ return new
VarcharLiteral(DateUtils.dateTimeFormatterChecklength(format.getValue(),
datetime).format(
java.time.LocalDateTime.of(((int) date.getYear()), ((int)
date.getMonth()), ((int) date.getDay()),
- ((int) date.getHour()), ((int) date.getMinute()),
((int) date.getSecond()))));
+ ((int) date.getHour()), ((int) date.getMinute()),
((int) date.getSecond()),
+ ((int) date.getMicroSecond() * 1000))));
}
+ /**
+ * datetime arithmetic function date-format
+ */
@ExecFunction(name = "date_format")
public static Expression dateFormat(DateV2Literal date, StringLikeLiteral
format) {
+ if (StringUtils.trim(format.getValue()).length() > 128) {
+ throw new AnalysisException("The length of format string in
date_format() function should not be greater"
+ + " than 128.");
+ }
+ DateTimeV2Literal datetime = new DateTimeV2Literal(date.getYear(),
date.getMonth(), date.getDay(), 0, 0, 0, 0);
format = (StringLikeLiteral)
SupportJavaDateFormatter.translateJavaFormatter(format);
- return new
VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()).format(
+ return new
VarcharLiteral(DateUtils.dateTimeFormatterChecklength(format.getValue(),
datetime).format(
java.time.LocalDate.of(((int) date.getYear()), ((int)
date.getMonth()), ((int) date.getDay()))));
}
+ /**
+ * datetime arithmetic function date-format
+ */
@ExecFunction(name = "date_format")
public static Expression dateFormat(DateTimeV2Literal date,
StringLikeLiteral format) {
+ if (StringUtils.trim(format.getValue()).length() > 128) {
+ throw new AnalysisException("The length of format string in
date_format() function should not be greater"
+ + " than 128.");
+ }
format = (StringLikeLiteral)
SupportJavaDateFormatter.translateJavaFormatter(format);
- return new
VarcharLiteral(DateUtils.dateTimeFormatter(format.getValue()).format(
+ return new
VarcharLiteral(DateUtils.dateTimeFormatterChecklength(format.getValue(),
date).format(
java.time.LocalDateTime.of(((int) date.getYear()), ((int)
date.getMonth()), ((int) date.getDay()),
((int) date.getHour()), ((int) date.getMinute()),
((int) date.getSecond()),
((int) date.getMicroSecond() * 1000))));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java
index 047b8d43e10..ea05d2d3c1a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.util;
import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableSet;
@@ -46,6 +47,204 @@ public class DateUtils {
"%Y-%m-%d %T", "%Y%m%d", "%Y%m");
private static final WeekFields weekFields =
WeekFields.of(DayOfWeek.SUNDAY, 7);
+ /**
+ * get the length of day of week string
+ */
+ public static int dayOfWeekLength(int dayOfWeek) {
+ if (dayOfWeek == 1 || dayOfWeek == 7 || dayOfWeek == 5) {
+ return 6;
+ } else if (dayOfWeek == 2) {
+ return 7;
+ } else if (dayOfWeek == 6 || dayOfWeek == 4) {
+ return 8;
+ } else {
+ return 10;
+ }
+ }
+
+ /**
+ * get the length of month string
+ */
+ public static int monthLength(long month) {
+ if (month == 5) {
+ return 3;
+ } else if (month == 6 || month == 7) {
+ return 4;
+ } else if (month == 4 || month == 3) {
+ return 5;
+ } else if (month == 8) {
+ return 6;
+ } else if (month == 10 || month == 1) {
+ return 7;
+ } else if (month == 2 || month == 11 || month == 12) {
+ return 8;
+ } else {
+ return 9;
+ }
+ }
+
+ /**
+ * format builder with length check.
+ * for some pattern, the length of result string is not fixed,
+ * so we need to check the length of result string to avoid parse error.
+ */
+ public static DateTimeFormatter dateTimeFormatterChecklength(String
pattern, DateTimeV2Literal date)
+ throws AnalysisException {
+ DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
+ boolean escaped = false;
+ int length = 0;
+ for (int i = 0; i < pattern.length(); i++) {
+ char character = pattern.charAt(i);
+ if (length > 100) {
+ throw new AnalysisException("Date format string is too long");
+ }
+ if (escaped) {
+ switch (character) {
+ case 'a': // %a Abbreviated weekday name (Sun..Sat)
+ builder.appendText(ChronoField.DAY_OF_WEEK,
TextStyle.SHORT);
+ length += 3;
+ break;
+ case 'b': // %b Abbreviated month name (Jan..Dec)
+ builder.appendText(ChronoField.MONTH_OF_YEAR,
TextStyle.SHORT);
+ length += 3;
+ break;
+ case 'c': // %c Month, numeric (0..12)
+ builder.appendValue(ChronoField.MONTH_OF_YEAR);
+ if (date.getMonth() < 10) {
+ length += 1;
+ } else {
+ length += 2;
+ }
+ break;
+ case 'd': // %d Day of the month, numeric (00..31)
+ builder.appendValue(ChronoField.DAY_OF_MONTH, 2);
+ length += 2;
+ break;
+ case 'e': // %e Day of the month, numeric (0..31)
+ builder.appendValue(ChronoField.DAY_OF_MONTH);
+ if (date.getDay() < 10) {
+ length += 1;
+ } else {
+ length += 2;
+ }
+ break;
+ case 'H': // %H Hour (00..23)
+ builder.appendValue(ChronoField.HOUR_OF_DAY, 2);
+ length += 2;
+ break;
+ case 'h': // %h Hour (01..12)
+ case 'I': // %I Hour (01..12)
+ builder.appendValue(ChronoField.CLOCK_HOUR_OF_AMPM, 2);
+ length += 2;
+ break;
+ case 'i': // %i Minutes, numeric (00..59)
+ builder.appendValue(ChronoField.MINUTE_OF_HOUR, 2);
+ length += 2;
+ break;
+ case 'j': // %j Day of year (001..366)
+ builder.appendValue(ChronoField.DAY_OF_YEAR, 3);
+ length += 3;
+ break;
+ case 'k': // %k Hour (0..23)
+ builder.appendValue(ChronoField.HOUR_OF_DAY);
+ length += 2;
+ break;
+ case 'l': // %l Hour (1..12)
+ builder.appendValue(ChronoField.CLOCK_HOUR_OF_AMPM);
+ if ((date.getHour() % 24 + 11) % 12 + 1 >= 10) {
+ length += 2;
+ } else {
+ length += 1;
+ }
+ break;
+ case 'M': // %M Month name (January..December)
+ builder.appendText(ChronoField.MONTH_OF_YEAR,
TextStyle.FULL);
+ length += monthLength(date.getMonth());
+ break;
+ case 'm': // %m Month, numeric (00..12）
+ builder.appendValue(ChronoField.MONTH_OF_YEAR, 2);
+ length += 2;
+ break;
+ case 'p': // %p AM or PM
+ builder.appendText(ChronoField.AMPM_OF_DAY);
+ length += 2;
+ break;
+ case 'r': // %r Time, 12-hour (hh:mm:ss followed by AM or
PM)
+ builder.appendValue(ChronoField.CLOCK_HOUR_OF_AMPM, 2)
+ .appendLiteral(':')
+ .appendValue(ChronoField.MINUTE_OF_HOUR, 2)
+ .appendLiteral(':')
+ .appendValue(ChronoField.SECOND_OF_MINUTE, 2)
+ .appendLiteral(' ')
+ .appendText(ChronoField.AMPM_OF_DAY,
TextStyle.FULL)
+ .toFormatter();
+ length += 11;
+ break;
+ case 'S': // %S Seconds (00..59)
+ case 's': // %s Seconds (00..59)
+ builder.appendValue(ChronoField.SECOND_OF_MINUTE, 2);
+ length += 2;
+ break;
+ case 'T': // %T Time, 24-hour (HH:mm:ss)
+ builder.appendPattern("HH:mm:ss");
+ length += 8;
+ break;
+ case 'V': // %V Week (01..53), where Sunday is the first
day of the week; used with %X
+ builder.appendValue(weekFields.weekOfWeekBasedYear(),
2);
+ length += 2;
+ break;
+ case 'v': // %v Week (01..53), where Monday is the first
day of the week; used with %x
+ length += 2;
+ builder.appendValue(IsoFields.WEEK_OF_WEEK_BASED_YEAR,
2);
+ break;
+ case 'W': // %W Weekday name (Sunday..Saturday)
+ builder.appendText(ChronoField.DAY_OF_WEEK,
TextStyle.FULL);
+ length += dayOfWeekLength(date.getDayOfWeek());
+ break;
+ case 'x': // %x Year for the week where Monday is the
first day of the week,
+ builder.appendValue(IsoFields.WEEK_BASED_YEAR, 4);
+ length += 4;
+ break;
+ case 'X':
+ builder.appendValue(weekFields.weekBasedYear(), 4, 10,
SignStyle.EXCEEDS_PAD);
+ length += 4;
+ break;
+ case 'Y': // %Y Year, numeric, four digits
+ // %X Year for the week, where Sunday is the first day
of the week,
+ // numeric, four digits; used with %v
+ builder.appendValue(ChronoField.YEAR, 4);
+ length += 4;
+ break;
+ case 'y': // %y Year, numeric (two digits)
+ builder.appendValueReduced(ChronoField.YEAR, 2, 2,
1970);
+ length += 2;
+ break;
+ // TODO(Gabriel): support microseconds in date literal
+ case 'D': // %D Day of the month with English suffix (0th,
1st, 2nd, 3rd, …)
+ case 'f': // %f Microseconds (000000..999999)
+ case 'U': // %U Week (00..53), where Sunday is the first
day of the week
+ case 'u': // %u Week (00..53), where Monday is the first
day of the week
+ case 'w': // %w Day of the week (0=Sunday..6=Saturday)
+ throw new AnalysisException(String.format("%%%s not
supported in date format string",
+ character));
+ case '%': // %% A literal "%" character
+ builder.appendLiteral('%');
+ break;
+ default: // %<x> The literal character represented by <x>
+ builder.appendLiteral(character);
+ length += 1;
+ break;
+ }
+ escaped = false;
+ } else if (character == '%') {
+ escaped = true;
+ } else {
+ builder.appendLiteral(character);
+ }
+ }
+ return
builder.toFormatter(Locale.US).withResolverStyle(ResolverStyle.STRICT);
+ }
+
/**
* format builder.
*/
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
index 570534584ec..0ba1f18c054 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
@@ -128,6 +128,7 @@ import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.util.MemoTestUtils;
import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -432,6 +433,11 @@ class FoldConstantTest extends ExpressionRewriteTestHelper
{
rewritten = executor.rewrite(d, context);
Assertions.assertEquals(new VarcharLiteral("01 01 01"), rewritten);
+ d = new DateFormat(DateV2Literal.fromJavaDateType(LocalDateTime.of(1,
1, 1, 1, 1, 1)),
+ StringLiteral.of(StringUtils.repeat("s", 128) + "
"));
+
+ rewritten = executor.rewrite(d, context);
+ Assertions.assertEquals(new VarcharLiteral(StringUtils.repeat("s",
128) + " "), rewritten);
DateTrunc t = new
DateTrunc(DateTimeV2Literal.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1,
1)),
StringLiteral.of("week"));
rewritten = executor.rewrite(t, context);
diff --git
a/regression-test/data/doc/sql-manual/sql-functions/doc_date_functions_test.out
b/regression-test/data/doc/sql-manual/sql-functions/doc_date_functions_test.out
index 3eea82fd705..7807b4c5e2b 100644
---
a/regression-test/data/doc/sql-manual/sql-functions/doc_date_functions_test.out
+++
b/regression-test/data/doc/sql-manual/sql-functions/doc_date_functions_test.out
@@ -113,6 +113,9 @@ Sunday October 2009
-- !date_format_4 --
01/01/1999 00:00:00
+-- !date_format_5 --
+111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111
+
-- !date_format_5 --
Sun Oct 10
diff --git
a/regression-test/suites/doc/sql-manual/sql-functions/doc_date_functions_test.groovy
b/regression-test/suites/doc/sql-manual/sql-functions/doc_date_functions_test.groovy
index 5e145b2b4b7..23f285ad291 100644
---
a/regression-test/suites/doc/sql-manual/sql-functions/doc_date_functions_test.groovy
+++
b/regression-test/suites/doc/sql-manual/sql-functions/doc_date_functions_test.groovy
@@ -154,6 +154,38 @@ suite("doc_date_functions_test") {
qt_date_format_2 """SELECT DATE_FORMAT('2007-10-04 22:23:00',
'%H:%i:%s')"""
qt_date_format_3 """SELECT DATE_FORMAT('1999-01-01', '%Y-%m-%d')"""
qt_date_format_4 """SELECT DATE_FORMAT('1999-01-01 00:00:00', '%d/%m/%Y
%H:%i:%s')"""
+
+ qt_date_format_5 """select date_format('2022-11-13
11:12:12',repeat('%l',51));"""
+
+ test {
+ sql """select date_format('2022-11-13 11:12:12',repeat('%l',52));"""
+ exception "Operation date_format of 142335814007783424,
%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l
is invalid"
+ }
+
+ test {
+ sql """select date_format('2023-11-13 23:00:00' ,repeat('%l',53));"""
+ exception "Operation date_format of 142406233473679360,
%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l%l
is invalid"
+ }
+
+ test {
+ sql """SELECT DATE_FORMAT('1222-12-12', repeat('s',129))"""
+ exception "Operation date_format of invalid or oversized format is
invalid"
+ }
+
+ test {
+ sql """select date_format('2022-11-13',repeat('%I',52));"""
+ exception "Operation date_format of 142335765945253888,
%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I%I
is invalid"
+ }
+
+ test {
+ sql """select date_format('2022-11-13',repeat('%M',15));"""
+ exception "Operation date_format of 142335765945253888,
%M%M%M%M%M%M%M%M%M%M%M%M%M%M%M is invalid"
+ }
+
+ test {
+ sql """select date_format('2022-11-13',repeat('%p',52));"""
+ exception "Operation date_format of 142335765945253888,
%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p%p
is invalid"
+ }
// 特殊格式符测试
qt_date_format_5 """SELECT DATE_FORMAT('2009-10-04', '%a %b %c')"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
