This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 092abb9dd85 [fix](fe) Align convert_tz folding with BE DST handling
(#64029)
092abb9dd85 is described below
commit 092abb9dd851adf8de8923cd3524ac4f68c6c49f
Author: feiniaofeiafei <[email protected]>
AuthorDate: Fri Jun 5 16:51:45 2026 +0800
[fix](fe) Align convert_tz folding with BE DST handling (#64029)
Related PR: #63853
Problem Summary:
FE constant folding for convert_tz() used Java LocalDateTime.atZone()
semantics. For source local times inside daylight-saving skipped
intervals, Java preserves the minute and second while shifting to the
post-transition offset, while BE executes convert_tz() through
cctz::convert() and maps skipped civil times to the transition instant.
This mismatch can make FE partition pruning reason about a different
folded value from BE execution. This PR makes FE convert_tz() constant
folding use ZoneRules transition information to match BE behavior for
skipped and repeated local times.
---
be/src/exprs/function/function_convert_tz.cpp | 21 ++++++++---
be/test/exprs/function/function_time_test.cpp | 17 +++++++++
.../executable/DateTimeExtractAndTransform.java | 6 ++-
.../trees/expressions/literal/DateTimeLiteral.java | 36 ++++++++++++++++--
.../DateTimeExtractAndTransformTest.java | 43 ++++++++++++++++++++++
.../literal/TimestampTzLiteralTest.java | 18 +++++++++
.../timestamptz/test_timestamptz_dst_gap.out | 4 +-
.../datetime_functions/test_convert_tz.out | 3 ++
.../datetime_functions/test_convert_tz.groovy | 2 +
9 files changed, 136 insertions(+), 14 deletions(-)
diff --git a/be/src/exprs/function/function_convert_tz.cpp
b/be/src/exprs/function/function_convert_tz.cpp
index 1c6932e2b66..aa2966cbc3f 100644
--- a/be/src/exprs/function/function_convert_tz.cpp
+++ b/be/src/exprs/function/function_convert_tz.cpp
@@ -212,6 +212,19 @@ private:
}
}
+ static std::pair<int64_t, int64_t> unix_timestamp_for_convert_tz(
+ const DateValueType& ts_value, const cctz::time_zone& from_tz) {
+ cctz::civil_second civil_time(ts_value.year(), ts_value.month(),
ts_value.day(),
+ ts_value.hour(), ts_value.minute(),
ts_value.second());
+ const auto lookup = from_tz.lookup(civil_time);
+ const bool skipped = lookup.kind ==
cctz::time_zone::civil_lookup::SKIPPED;
+ const auto tp = skipped ? lookup.trans : lookup.pre;
+
+ // Skipped civil times map to the transition instant. Do not keep the
+ // input fractional part inside a local time interval that never
existed.
+ return {tp.time_since_epoch().count(), skipped ? 0 :
ts_value.microsecond()};
+ }
+
static void execute_tz_const_with_state(ConvertTzState* convert_tz_state,
const ColumnType* date_column,
ColumnType* result_column,
NullMap& result_null_map,
@@ -239,9 +252,7 @@ private:
DateValueType ts_value = date_column->get_element(i);
DateValueType ts_value2;
- std::pair<int64_t, int64_t> timestamp;
- ts_value.unix_timestamp(×tamp, from_tz);
- ts_value2.from_unixtime(timestamp, to_tz);
+ ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value,
from_tz), to_tz);
if (!ts_value2.is_valid_date()) [[unlikely]] {
throw_out_of_bound_convert_tz<DateValueType>(date_column->get_element(i),
@@ -292,9 +303,7 @@ private:
to_tz_name);
}
- std::pair<int64_t, int64_t> timestamp;
- ts_value.unix_timestamp(×tamp, from_tz);
- ts_value2.from_unixtime(timestamp, to_tz);
+ ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value,
from_tz), to_tz);
if (!ts_value2.is_valid_date()) [[unlikely]] {
throw_out_of_bound_convert_tz<DateValueType>(date_column->get_element(index_now),
diff --git a/be/test/exprs/function/function_time_test.cpp
b/be/test/exprs/function/function_time_test.cpp
index 3043e65d592..e012c7784e9 100644
--- a/be/test/exprs/function/function_time_test.cpp
+++ b/be/test/exprs/function/function_time_test.cpp
@@ -326,6 +326,23 @@ TEST(VTimestampFunctionsTest, convert_tz_test) {
check_function<DataTypeDateTimeV2, true>(func_name,
input_types, data_set));
}
+ {
+ InputTypeSet input_types_scale6 = {{PrimitiveType::TYPE_DATETIMEV2, 6},
+ PrimitiveType::TYPE_VARCHAR,
+ PrimitiveType::TYPE_VARCHAR};
+ DataSet data_set = {{{std::string {"2021-03-28 02:15:30.123456"},
+ std::string {"Europe/Paris"}, std::string
{"UTC"}},
+ std::string("2021-03-28 01:00:00.000000")},
+ {{std::string {"2021-03-28 03:00:30.123456"},
+ std::string {"Europe/Paris"}, std::string
{"UTC"}},
+ std::string("2021-03-28 01:00:30.123456")},
+ {{std::string {"2021-10-31 02:15:30.123456"},
+ std::string {"Europe/Paris"}, std::string
{"UTC"}},
+ std::string("2021-10-31 00:15:30.123456")}};
+ static_cast<void>(check_function<DataTypeDateTimeV2, true>(func_name,
input_types_scale6,
+ data_set,
6));
+ }
+
{
DataSet data_set = {{{std::string {"2019-08-01 02:18:27"}, std::string
{"Asia/Shanghai"},
std::string {"UTC"}},
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java
index acbdfc25504..f2324a396e7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java
@@ -26,6 +26,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.FromMilliseco
import org.apache.doris.nereids.trees.expressions.functions.scalar.FromSecond;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral;
@@ -814,8 +815,9 @@ public class DateTimeExtractAndTransform {
ZoneId toZone =
ZoneId.from(zoneFormatter.parse(toTz.getStringValue()));
LocalDateTime localDateTime = datetime.toJavaDateType();
- ZonedDateTime resultDateTime =
localDateTime.atZone(fromZone).withZoneSameInstant(toZone);
- return
DateTimeV2Literal.fromJavaDateType(resultDateTime.toLocalDateTime(),
datetime.getDataType().getScale());
+ Instant instant = DateTimeLiteral.convertLocalToInstant(localDateTime,
fromZone);
+ return
DateTimeV2Literal.fromJavaDateType(LocalDateTime.ofInstant(instant, toZone),
+ datetime.getDataType().getScale());
}
private static void validateTimezoneOffset(String timezone) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
index d6efcb57e3c..019e8d8a40f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
@@ -37,11 +37,16 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.math.BigInteger;
+import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
+import java.time.ZoneOffset;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQueries;
+import java.time.zone.ZoneOffsetTransition;
+import java.time.zone.ZoneRules;
+import java.util.List;
import java.util.Objects;
/**
@@ -271,10 +276,33 @@ public class DateTimeLiteral extends DateLiteral {
private static LocalDateTime convertTimeZone(long year, long month, long
day, long hour, long minute,
long second, ZoneId fromZone, ZoneId toZone) {
- return LocalDateTime.of((int) year, (int) month, (int) day, (int)
hour, (int) minute, (int) second)
- .atZone(fromZone)
- .withZoneSameInstant(toZone)
- .toLocalDateTime();
+ LocalDateTime localDateTime = LocalDateTime.of((int) year, (int)
month, (int) day,
+ (int) hour, (int) minute, (int) second);
+ Instant instant = convertLocalToInstant(localDateTime, fromZone);
+ return LocalDateTime.ofInstant(instant, toZone);
+ }
+
+ /**
+ * Convert a local civil datetime in {@code fromZone} to an instant with
the same DST transition
+ * policy as BE cctz::convert(civil_second, zone).
+ *
+ * <p>For normal local times, there is one valid offset. For fall-back
overlap times, two offsets
+ * are valid and the first one is the pre-transition offset. For
spring-forward gap times, the
+ * local time does not exist, so any value inside the skipped interval
maps to the transition
+ * instant.
+ */
+ public static Instant convertLocalToInstant(LocalDateTime localDateTime,
ZoneId fromZone) {
+ ZoneRules rules = fromZone.getRules();
+ List<ZoneOffset> validOffsets = rules.getValidOffsets(localDateTime);
+ int size = validOffsets.size();
+ // Match BE cctz::convert(civil_second, zone) semantics for constant
folding.
+ // Normal local time has one offset; repeated local time uses the
pre-transition offset.
+ if (size == 1 || size == 2) {
+ return localDateTime.atOffset(validOffsets.get(0)).toInstant();
+ }
+ // Skipped local time maps to the transition instant, e.g. 2021-03-28
02:15 Europe/Paris.
+ ZoneOffsetTransition transition = rules.getTransition(localDateTime);
+ return transition.getInstant();
}
public boolean checkRange() {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransformTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransformTest.java
index ee3604f83de..1423620f4eb 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransformTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransformTest.java
@@ -24,6 +24,7 @@ import
org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal;
import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.types.DateTimeV2Type;
import org.junit.jupiter.api.Assertions;
@@ -166,4 +167,46 @@ class DateTimeExtractAndTransformTest {
Assertions.assertThrows(AnalysisException.class,
() -> DateTimeExtractAndTransform.fromUnixTime(dec));
}
+
+ @Test
+ void testConvertTzDstTransition() {
+ // Spring gap maps skipped local times to the transition instant.
+ Assertions.assertEquals(
+ new DateTimeV2Literal("2021-03-28 01:00:00"),
+ DateTimeExtractAndTransform.convertTz(
+ new DateTimeV2Literal("2021-03-28 02:15:00"),
+ new VarcharLiteral("Europe/Paris"),
+ new VarcharLiteral("UTC")));
+ Assertions.assertEquals(
+ new DateTimeV2Literal("2021-03-28 01:00:00"),
+ DateTimeExtractAndTransform.convertTz(
+ new DateTimeV2Literal("2021-03-28 02:00:00"),
+ new VarcharLiteral("Europe/Paris"),
+ new VarcharLiteral("UTC")));
+ Assertions.assertEquals(
+ new DateTimeV2Literal("2021-03-28 01:00:00"),
+ DateTimeExtractAndTransform.convertTz(
+ new DateTimeV2Literal("2021-03-28 03:00:00"),
+ new VarcharLiteral("Europe/Paris"),
+ new VarcharLiteral("UTC")));
+ // Fall overlap uses the pre-transition offset.
+ Assertions.assertEquals(
+ new DateTimeV2Literal("2021-10-31 00:15:00"),
+ DateTimeExtractAndTransform.convertTz(
+ new DateTimeV2Literal("2021-10-31 02:15:00"),
+ new VarcharLiteral("Europe/Paris"),
+ new VarcharLiteral("UTC")));
+ Assertions.assertEquals(
+ new DateTimeV2Literal("2021-10-31 00:00:00"),
+ DateTimeExtractAndTransform.convertTz(
+ new DateTimeV2Literal("2021-10-31 02:00:00"),
+ new VarcharLiteral("Europe/Paris"),
+ new VarcharLiteral("UTC")));
+ Assertions.assertEquals(
+ new DateTimeV2Literal("2021-10-31 02:00:00"),
+ DateTimeExtractAndTransform.convertTz(
+ new DateTimeV2Literal("2021-10-31 03:00:00"),
+ new VarcharLiteral("Europe/Paris"),
+ new VarcharLiteral("UTC")));
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java
index a1961a61fbb..3e000de8a82 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java
@@ -75,6 +75,24 @@ class TimestampTzLiteralTest {
Assertions.assertEquals(0, literal.minute);
Assertions.assertEquals(0, literal.second);
Assertions.assertEquals(0, literal.microSecond);
+
+ literal = new TimestampTzLiteral(TimeStampTzType.of(6), "2024-03-10
02:30:00 America/New_York");
+ Assertions.assertEquals(2024, literal.year);
+ Assertions.assertEquals(3, literal.month);
+ Assertions.assertEquals(10, literal.day);
+ Assertions.assertEquals(7, literal.hour);
+ Assertions.assertEquals(0, literal.minute);
+ Assertions.assertEquals(0, literal.second);
+ Assertions.assertEquals(0, literal.microSecond);
+
+ literal = new TimestampTzLiteral(TimeStampTzType.of(6), "2024-11-03
01:30:00 America/New_York");
+ Assertions.assertEquals(2024, literal.year);
+ Assertions.assertEquals(11, literal.month);
+ Assertions.assertEquals(3, literal.day);
+ Assertions.assertEquals(5, literal.hour);
+ Assertions.assertEquals(30, literal.minute);
+ Assertions.assertEquals(0, literal.second);
+ Assertions.assertEquals(0, literal.microSecond);
}
@Test
diff --git
a/regression-test/data/datatype_p0/timestamptz/test_timestamptz_dst_gap.out
b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_dst_gap.out
index ae3b0c924d0..01c3cdfbdf2 100644
--- a/regression-test/data/datatype_p0/timestamptz/test_timestamptz_dst_gap.out
+++ b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_dst_gap.out
@@ -1,9 +1,9 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !sql --
-1 named_gap_ny 2024-03-10 07:30:00.000000+00:00
+1 named_gap_ny 2024-03-10 07:00:00.000000+00:00
2 explicit_before_gap 2024-03-10 06:30:00.000000+00:00
3 explicit_after_gap 2024-03-10 07:30:00.000000+00:00
-4 implicit_gap_ny 2024-03-10 07:30:00.000000+00:00
+4 implicit_gap_ny 2024-03-10 07:00:00.000000+00:00
5 implicit_after_gap 2024-03-10 07:30:00.000000+00:00
-- !sql --
diff --git
a/regression-test/data/query_p0/sql_functions/datetime_functions/test_convert_tz.out
b/regression-test/data/query_p0/sql_functions/datetime_functions/test_convert_tz.out
index 9bf49bd74ef..643540d9c7b 100644
---
a/regression-test/data/query_p0/sql_functions/datetime_functions/test_convert_tz.out
+++
b/regression-test/data/query_p0/sql_functions/datetime_functions/test_convert_tz.out
@@ -21,3 +21,6 @@
2024-04-18T23:20
2024-04-18T23:20
+-- !spring_gp_with_micro_sec --
+2021-03-28T01:00
+
diff --git
a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy
b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy
index e382fe9ae70..c9f37b37b4f 100644
---
a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy
+++
b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy
@@ -35,4 +35,6 @@ suite("test_convert_tz") {
order_qt_sql1 """
select convert_tz(dt, '+00:00', IF(property_value IS NULL, '+00:00',
property_value)) from cvt_tz
"""
+ sql "set debug_skip_fold_constant=true;"
+ qt_spring_gp_with_micro_sec "select convert_tz('2021-03-28
02:30:00.00323', 'Europe/Paris','UTC');"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
