This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit b143f0dfe2cb269ae8de8b8723d6077b432553b9 Author: Pxl <[email protected]> AuthorDate: Fri May 24 22:23:55 2024 +0800 [Improvement](date) shortcut for str to date parse (#35288) shortcut for str to date parse --- .../function_date_or_datetime_computation.h | 58 +---------- be/src/vec/runtime/vdatetime_value.cpp | 34 +++---- be/src/vec/runtime/vdatetime_value.h | 99 ------------------- be/test/vec/runtime/vdatetime_value_test.cpp | 106 --------------------- 4 files changed, 18 insertions(+), 279 deletions(-) diff --git a/be/src/vec/functions/function_date_or_datetime_computation.h b/be/src/vec/functions/function_date_or_datetime_computation.h index 6c707c4b1ca..92a70bc8c92 100644 --- a/be/src/vec/functions/function_date_or_datetime_computation.h +++ b/be/src/vec/functions/function_date_or_datetime_computation.h @@ -288,7 +288,6 @@ struct TimeDiffImpl { using ReturnType = DataTypeTimeV2; static constexpr auto name = "timediff"; - static constexpr auto is_nullable = false; static constexpr int64_t limit_value = 3020399000000; // 838:59:59 convert to microsecond static inline ReturnType::FieldType execute(const ArgType1& t0, const ArgType2& t1, bool& is_null) { @@ -749,56 +748,11 @@ public: WhichDataType which1(remove_nullable(first_arg_type)); WhichDataType which2(remove_nullable(second_arg_type)); - if (which1.is_date() && which2.is_date()) { - return DateTimeAddIntervalImpl<DataTypeDate::FieldType, Transform, - DataTypeDate::FieldType>::execute(block, arguments, - result, - input_rows_count); - } else if (which1.is_date_time() && which2.is_date()) { - return DateTimeAddIntervalImpl<DataTypeDateTime::FieldType, Transform, - DataTypeDate::FieldType>::execute(block, arguments, - result, - input_rows_count); - } else if (which1.is_date_v2() && which2.is_date()) { + if (which1.is_date_v2() && which2.is_date_v2()) { return DateTimeAddIntervalImpl<DataTypeDateV2::FieldType, Transform, - DataTypeDate::FieldType>::execute(block, arguments, - result, - input_rows_count); - } else if (which1.is_date_time_v2() && which2.is_date()) { - return DateTimeAddIntervalImpl<DataTypeDateTimeV2::FieldType, Transform, - DataTypeDate::FieldType>::execute(block, arguments, - result, - input_rows_count); - } else if (which1.is_date() && which2.is_date_time()) { - return DateTimeAddIntervalImpl<DataTypeDate::FieldType, Transform, - DataTypeDateTime::FieldType>::execute(block, arguments, - result, - input_rows_count); - } else if (which1.is_date() && which2.is_date_v2()) { - return DateTimeAddIntervalImpl<DataTypeDate::FieldType, Transform, DataTypeDateV2::FieldType>::execute(block, arguments, result, input_rows_count); - } else if (which1.is_date() && which2.is_date_time_v2()) { - return DateTimeAddIntervalImpl< - DataTypeDate::FieldType, Transform, - DataTypeDateTimeV2::FieldType>::execute(block, arguments, result, - input_rows_count); - } else if (which1.is_date_v2() && which2.is_date_time()) { - return DateTimeAddIntervalImpl<DataTypeDateV2::FieldType, Transform, - DataTypeDateTime::FieldType>::execute(block, arguments, - result, - input_rows_count); - } else if (which1.is_date_v2() && which2.is_date_v2()) { - return DateTimeAddIntervalImpl<DataTypeDateV2::FieldType, Transform, - DataTypeDateV2::FieldType>::execute(block, arguments, - result, - input_rows_count); - } else if (which1.is_date_time_v2() && which2.is_date_time()) { - return DateTimeAddIntervalImpl<DataTypeDateTimeV2::FieldType, Transform, - DataTypeDateTime::FieldType>::execute(block, arguments, - result, - input_rows_count); } else if (which1.is_date_time_v2() && which2.is_date_time_v2()) { return DateTimeAddIntervalImpl< DataTypeDateTimeV2::FieldType, Transform, @@ -809,16 +763,6 @@ public: DataTypeDateTime::FieldType>::execute(block, arguments, result, input_rows_count); - } else if (which1.is_date_time() && which2.is_date_v2()) { - return DateTimeAddIntervalImpl<DataTypeDateTime::FieldType, Transform, - DataTypeDateV2::FieldType>::execute(block, arguments, - result, - input_rows_count); - } else if (which1.is_date_time() && which2.is_date_time_v2()) { - return DateTimeAddIntervalImpl< - DataTypeDateTime::FieldType, Transform, - DataTypeDateTimeV2::FieldType>::execute(block, arguments, result, - input_rows_count); } else if (which1.is_date_v2() && which2.is_date_time_v2()) { return DateTimeAddIntervalImpl< DataTypeDateV2::FieldType, Transform, diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp index 5c6e788695c..bd2a7044a7f 100644 --- a/be/src/vec/runtime/vdatetime_value.cpp +++ b/be/src/vec/runtime/vdatetime_value.cpp @@ -2027,6 +2027,7 @@ bool DateV2Value<T>::from_date_str_base(const char* date_str, int len, int scale int field_idx = 0; int field_len = year_len; long sec_offset = 0; + bool need_use_timezone = false; while (ptr < end && isdigit(*ptr) && field_idx < MAX_DATE_PARTS) { const char* start = ptr; @@ -2037,6 +2038,10 @@ bool DateV2Value<T>::from_date_str_base(const char* date_str, int len, int scale ptr++; } + if (ptr == start) { + return false; + } + if (field_idx == 6) { if constexpr (is_datetime) { // round of microseconds @@ -2107,23 +2112,7 @@ bool DateV2Value<T>::from_date_str_base(const char* date_str, int len, int scale if (local_time_zone == nullptr) { return false; } - auto get_tz_offset = [&](const std::string& str_tz, - const cctz::time_zone* local_time_zone) -> long { - cctz::time_zone given_tz {}; - if (!TimezoneUtils::find_cctz_time_zone(str_tz, given_tz)) { - throw Exception {ErrorCode::INVALID_ARGUMENT, ""}; - } - auto given = cctz::convert(cctz::civil_second {}, given_tz); - auto local = cctz::convert(cctz::civil_second {}, *local_time_zone); - // these two values is absolute time. so they are negative. need to use (-local) - (-given) - return std::chrono::duration_cast<std::chrono::seconds>(given - local).count(); - }; - try { - sec_offset = get_tz_offset(std::string {ptr, end}, - local_time_zone); // use the whole remain string - } catch ([[maybe_unused]] Exception& e) { - return false; // invalid format - } + need_use_timezone = true; field_idx++; break; } @@ -2197,6 +2186,17 @@ bool DateV2Value<T>::from_date_str_base(const char* date_str, int len, int scale } } + if (need_use_timezone) { + cctz::time_zone given_tz {}; + if (!TimezoneUtils::find_cctz_time_zone(std::string {ptr, end}, given_tz)) { + return false; // invalid format + } + auto given = cctz::convert(cctz::civil_second {}, given_tz); + auto local = cctz::convert(cctz::civil_second {}, *local_time_zone); + // these two values is absolute time. so they are negative. need to use (-local) - (-given) + sec_offset = std::chrono::duration_cast<std::chrono::seconds>(given - local).count(); + } + // In check_range_and_set_time, for Date type the time part will be truncated. So if the timezone offset should make // rounding to date part, it would be lost. To avoid this, we use a Datetime type to do these calc. It will save the // time part and apply the offset. Then convert to Date type back. diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index dfc3ab332a7..0251964b809 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -1483,105 +1483,6 @@ int64_t datetime_diff(const DateV2Value<T0>& ts_value1, const DateV2Value<T1>& t return 0; } -template <TimeUnit unit, typename T> -int64_t datetime_diff(const DateV2Value<T>& ts_value1, const VecDateTimeValue& ts_value2) { - // FIXME: - switch (unit) { - case YEAR: { - int year = (ts_value2.year() - ts_value1.year()); - if (year > 0) { - year -= ts_value1.month() - ts_value2.month() < 0; - } else if (year < 0) { - year += ts_value1.month() - ts_value2.month() > 0; - } - return year; - } - case MONTH: { - int month = (ts_value2.year() - ts_value1.year()) * 12 + - (ts_value2.month() - ts_value1.month()); - if (month > 0) { - month -= (ts_value2.day() - ts_value1.day()) < 0; - } else if (month < 0) { - month += (ts_value2.day() - ts_value1.day()) > 0; - } - return month; - } - case WEEK: { - int day = ts_value2.daynr() - ts_value1.daynr(); - return day / 7; - } - case DAY: { - int day = ts_value2.daynr() - ts_value1.daynr(); - return day; - } - case HOUR: { - int64_t second = ts_value2.second_diff(ts_value1); - int64_t hour = second / 60 / 60; - return hour; - } - case MINUTE: { - int64_t second = ts_value2.second_diff(ts_value1); - int64_t minute = second / 60; - return minute; - } - case SECOND: { - int64_t second = ts_value2.second_diff(ts_value1); - return second; - } - } - // Rethink the default return value - return 0; -} - -template <TimeUnit unit, typename T> -int64_t datetime_diff(const VecDateTimeValue& ts_value1, const DateV2Value<T>& ts_value2) { - switch (unit) { - case YEAR: { - int year = (ts_value2.year() - ts_value1.year()); - if (year > 0) { - year -= ts_value1.month() - ts_value2.month() < 0; - } else if (year < 0) { - year -= ts_value1.month() - ts_value2.month() > 0; - } - return year; - } - case MONTH: { - int month = (ts_value2.year() - ts_value1.year()) * 12 + - (ts_value2.month() - ts_value1.month()); - if (month > 0) { - month -= (ts_value2.day() - ts_value1.day()) < 0; - } else if (month < 0) { - month += (ts_value2.day() - ts_value1.day()) > 0; - } - return month; - } - case WEEK: { - int day = ts_value2.daynr() - ts_value1.daynr(); - return day / 7; - } - case DAY: { - int day = ts_value2.daynr() - ts_value1.daynr(); - return day; - } - case HOUR: { - int64_t second = ts_value2.second_diff(ts_value1); - int64_t hour = second / 60 / 60; - return hour; - } - case MINUTE: { - int64_t second = ts_value2.second_diff(ts_value1); - int64_t minute = second / 60; - return minute; - } - case SECOND: { - int64_t second = ts_value2.second_diff(ts_value1); - return second; - } - } - // Rethink the default return value - return 0; -} - /** * Date dict table. date range is [1900-01-01, 2039-12-31]. */ diff --git a/be/test/vec/runtime/vdatetime_value_test.cpp b/be/test/vec/runtime/vdatetime_value_test.cpp index bb396b2ce6f..2cb67a9e2e7 100644 --- a/be/test/vec/runtime/vdatetime_value_test.cpp +++ b/be/test/vec/runtime/vdatetime_value_test.cpp @@ -335,112 +335,6 @@ TEST(VDateTimeValueTest, date_diff_test) { 31 * 24 * 60 * 60 + 3601); } - { - DateV2Value<DateV2ValueType> date_v2_1; - std::string origin_date1 = "2022-05-24"; - std::string date_format1 = "%Y-%m-%d"; - EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(), - origin_date1.data(), origin_date1.size())); - - VecDateTimeValue date_v2_2; - std::string origin_date2 = "2022-06-24 00:00:00"; - std::string date_format2 = "%Y-%m-%d %H:%i:%s"; - EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(), - origin_date2.data(), origin_date2.size())); - - EXPECT_TRUE(datetime_diff<TimeUnit::DAY>(date_v2_1, date_v2_2) == 31); - EXPECT_TRUE(datetime_diff<TimeUnit::YEAR>(date_v2_1, date_v2_2) == 0); - EXPECT_TRUE(datetime_diff<TimeUnit::MONTH>(date_v2_1, date_v2_2) == 1); - EXPECT_TRUE(datetime_diff<TimeUnit::HOUR>(date_v2_1, date_v2_2) == 31 * 24); - EXPECT_TRUE(datetime_diff<TimeUnit::MINUTE>(date_v2_1, date_v2_2) == 31 * 24 * 60); - EXPECT_TRUE(datetime_diff<TimeUnit::SECOND>(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60); - } - - { - DateV2Value<DateTimeV2ValueType> date_v2_1; - std::string origin_date1 = "2022-05-24 00:00:00.000"; - std::string date_format1 = "%Y-%m-%d %H:%i:%s.%f"; - EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(), - origin_date1.data(), origin_date1.size())); - - VecDateTimeValue date_v2_2; - std::string origin_date2 = "2022-06-24 00:00:00"; - std::string date_format2 = "%Y-%m-%d %H:%i:%s"; - EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(), - origin_date2.data(), origin_date2.size())); - - EXPECT_TRUE(datetime_diff<TimeUnit::DAY>(date_v2_1, date_v2_2) == 31); - EXPECT_TRUE(datetime_diff<TimeUnit::YEAR>(date_v2_1, date_v2_2) == 0); - EXPECT_TRUE(datetime_diff<TimeUnit::MONTH>(date_v2_1, date_v2_2) == 1); - EXPECT_TRUE(datetime_diff<TimeUnit::HOUR>(date_v2_1, date_v2_2) == 31 * 24); - EXPECT_TRUE(datetime_diff<TimeUnit::MINUTE>(date_v2_1, date_v2_2) == 31 * 24 * 60); - EXPECT_TRUE(datetime_diff<TimeUnit::SECOND>(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60); - } - - { - VecDateTimeValue date_v2_1; - std::string origin_date1 = "2022-05-24 00:00:00"; - std::string date_format1 = "%Y-%m-%d %H:%i:%s"; - EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(), - origin_date1.data(), origin_date1.size())); - - DateV2Value<DateV2ValueType> date_v2_2; - std::string origin_date2 = "2022-06-24"; - std::string date_format2 = "%Y-%m-%d"; - EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(), - origin_date2.data(), origin_date2.size())); - - EXPECT_TRUE(datetime_diff<TimeUnit::DAY>(date_v2_1, date_v2_2) == 31); - EXPECT_TRUE(datetime_diff<TimeUnit::YEAR>(date_v2_1, date_v2_2) == 0); - EXPECT_TRUE(datetime_diff<TimeUnit::MONTH>(date_v2_1, date_v2_2) == 1); - EXPECT_TRUE(datetime_diff<TimeUnit::HOUR>(date_v2_1, date_v2_2) == 31 * 24); - EXPECT_TRUE(datetime_diff<TimeUnit::MINUTE>(date_v2_1, date_v2_2) == 31 * 24 * 60); - EXPECT_TRUE(datetime_diff<TimeUnit::SECOND>(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60); - } - - { - VecDateTimeValue date_v2_1; - std::string origin_date1 = "2022-05-24 00:00:00"; - std::string date_format1 = "%Y-%m-%d %H:%i:%s"; - EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(), - origin_date1.data(), origin_date1.size())); - - DateV2Value<DateTimeV2ValueType> date_v2_2; - std::string origin_date2 = "2022-06-24 00:00:00.000000"; - std::string date_format2 = "%Y-%m-%d %H:%i:%s.%f"; - EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(), - origin_date2.data(), origin_date2.size())); - - EXPECT_TRUE(datetime_diff<TimeUnit::DAY>(date_v2_1, date_v2_2) == 31); - EXPECT_TRUE(datetime_diff<TimeUnit::YEAR>(date_v2_1, date_v2_2) == 0); - EXPECT_TRUE(datetime_diff<TimeUnit::MONTH>(date_v2_1, date_v2_2) == 1); - EXPECT_TRUE(datetime_diff<TimeUnit::HOUR>(date_v2_1, date_v2_2) == 31 * 24); - EXPECT_TRUE(datetime_diff<TimeUnit::MINUTE>(date_v2_1, date_v2_2) == 31 * 24 * 60); - EXPECT_TRUE(datetime_diff<TimeUnit::SECOND>(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60); - } - - { - DateV2Value<DateV2ValueType> date_v2_1; - std::string origin_date1 = "2022-05-24"; - std::string date_format1 = "%Y-%m-%d"; - EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(), - origin_date1.data(), origin_date1.size())); - - VecDateTimeValue date_v2_2; - std::string origin_date2 = "2022-06-24 06:00:00"; - std::string date_format2 = "%Y-%m-%d %H:%i:%s"; - EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(), - origin_date2.data(), origin_date2.size())); - - EXPECT_TRUE(datetime_diff<TimeUnit::DAY>(date_v2_1, date_v2_2) == 31); - EXPECT_TRUE(datetime_diff<TimeUnit::YEAR>(date_v2_1, date_v2_2) == 0); - EXPECT_TRUE(datetime_diff<TimeUnit::MONTH>(date_v2_1, date_v2_2) == 1); - EXPECT_TRUE(datetime_diff<TimeUnit::HOUR>(date_v2_1, date_v2_2) == 31 * 24 + 6); - EXPECT_TRUE(datetime_diff<TimeUnit::MINUTE>(date_v2_1, date_v2_2) == (31 * 24 + 6) * 60); - EXPECT_TRUE(datetime_diff<TimeUnit::SECOND>(date_v2_1, date_v2_2) == - (31 * 24 + 6) * 60 * 60); - } - { VecDateTimeValue date_v2_1; std::string origin_date1 = "2022-05-24"; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
