github-actions[bot] commented on code in PR #70205:
URL: https://github.com/apache/doris/pull/70205#discussion_r4611087751
##########
be/src/format/table/es/es_scroll_parser.cpp:
##########
@@ -427,103 +427,123 @@ static bool set_es_epoch_seconds(DateValue& value,
int64_t seconds,
return true;
}
-template <PrimitiveType T>
-Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type,
bool is_date_str,
- typename PrimitiveTypeTraits<T>::CppType* slot,
- const cctz::time_zone& time_zone) {
- constexpr bool is_datetime_v1 = T == TYPE_DATE || T == TYPE_DATETIME;
- typename PrimitiveTypeTraits<T>::CppType dt_val;
- if (is_date_str) {
- std::string str_date(col.GetString(), col.GetStringLength());
- size_t str_length = col.GetStringLength();
- // The default strict_date_optional_time parser wins over epoch_millis
for a four-digit year.
- // Thus "2024" means 2024-01-01, whereas "12345" means 12.345 seconds
after the epoch.
- if (str_length == 4 && str_date.find_first_not_of("0123456789") ==
std::string::npos) {
- str_date += "-01-01";
- str_length = str_date.size();
+static bool es_calendar_format_accepts(const std::string& text,
std::string_view format,
+ bool* year_only) {
+ const bool digits = !text.empty() && text.find_first_not_of("0123456789")
== std::string::npos;
+ *year_only = false;
+ // Compact dates win only for their mapped grammar. "20240101" is an epoch
in the default
+ // format, whereas "2024" is a year only when an optional-date format
precedes epoch_millis.
+ if (format == "basic_date" || format == "yyyyMMdd") {
+ return digits && text.size() == 8;
+ }
+ if (format == "strict_date_optional_time" || format ==
"date_optional_time" ||
+ format == "strict_date_optional_time_nanos") {
+ if (digits) {
+ *year_only = text.size() == 4;
+ return *year_only;
}
- bool success = false;
- int64_t epoch_seconds;
- // Source fallback also reaches short, zero and fractional epoch
strings, e.g. "0"/"1.5".
- if (parse_es_epoch_millis(str_date, &epoch_seconds)) {
- success = set_es_epoch_seconds(dt_val, epoch_seconds, time_zone);
- } else if (str_length > 19) {
- std::chrono::system_clock::time_point tp;
- // time_zone suffix pattern
- // Z/+08:00/-04:30
- RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)");
- bool ok = false;
- std::string fmt;
- if (time_zone_pattern.Match(str_date, 0, str_date.size(),
RE2::UNANCHORED, nullptr,
- 0)) {
- // with time_zone info
- // YYYY-MM-DDTHH:MM:SSZ or YYYY-MM-DDTHH:MM:SS+08:00
- // or 2022-08-08T12:10:10.000Z or YYYY-MM-DDTHH:MM:SS-08:00
- fmt = "%Y-%m-%dT%H:%M:%E*S%Ez";
- ok = cctz::parse(fmt, str_date, cctz::utc_time_zone(), &tp);
- } else {
- // without time_zone info
- // 2022-08-08T12:10:10.000
- fmt = "%Y-%m-%dT%H:%M:%E*S";
- // If the time without time_zone info, ES will assume it is
UTC time.
- // So we parse it in Doris with UTC time zone.
- ok = cctz::parse(fmt, str_date, cctz::utc_time_zone(), &tp);
- }
- if (ok) {
- // The local time zone can change by session variable
`time_zone`
- // We should use the user specified time zone, not the actual
system local time zone.
- success = true;
- // A pre-epoch fractional instant must floor to its containing
second, like numeric dates.
- dt_val.from_unixtime(
-
std::chrono::floor<std::chrono::seconds>(tp).time_since_epoch().count(),
- time_zone);
- }
- } else if (str_length == 19) {
- // YYYY-MM-DDTHH:MM:SS
- if (*(str_date.c_str() + 10) == 'T') {
- std::chrono::system_clock::time_point tp;
- const bool ok =
- cctz::parse("%Y-%m-%dT%H:%M:%S", str_date,
cctz::utc_time_zone(), &tp);
- if (ok) {
- success = true;
-
dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), time_zone);
- }
- } else {
- // YYYY-MM-DD HH:MM:SS
- CastParameters params;
- if constexpr (is_datetime_v1) {
- success =
CastToDateOrDatetime::from_string_non_strict_mode<
- DatelikeTargetType::DATE_TIME>({str_date.c_str(),
str_length}, dt_val,
- nullptr, params);
- } else if constexpr (T == TYPE_DATEV2) {
- success = CastToDateV2::from_string_non_strict_mode(
- {str_date.c_str(), str_length}, dt_val, nullptr,
params);
- } else {
- success = CastToDatetimeV2::from_string_non_strict_mode(
- {str_date.c_str(), str_length}, dt_val, nullptr,
-1, params);
- }
- }
+ return true;
+ }
+ // Restrict named calendar shapes before trying a later epoch alternative.
+ if (format == "yyyy-MM-dd" || format == "strict_date" || format == "date")
{
+ return text.size() == 10 && text[4] == '-' && text[7] == '-';
+ }
+ if (format == "yyyy-MM-dd HH:mm:ss") {
+ return text.size() == 19 && text[4] == '-' && text[7] == '-' &&
text[10] == ' ';
+ }
+ return true;
+}
- } else {
- // YYYY-MM-DD or others
- CastParameters params;
- if constexpr (is_datetime_v1) {
- success = CastToDateOrDatetime::from_string_non_strict_mode<
- DatelikeTargetType::DATE_TIME>({str_date.c_str(),
str_length}, dt_val,
- nullptr, params);
- } else if constexpr (T == TYPE_DATEV2) {
- success =
CastToDateV2::from_string_non_strict_mode({str_date.c_str(), str_length},
- dt_val,
nullptr, params);
- } else {
- success = CastToDatetimeV2::from_string_non_strict_mode(
- {str_date.c_str(), str_length}, dt_val, nullptr, -1,
params);
+static bool parse_es_iso_datetime(const std::string& text, int64_t* seconds) {
+ std::chrono::system_clock::time_point time;
+ RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)");
+ bool success;
+ if (time_zone_pattern.Match(text, 0, text.size(), RE2::UNANCHORED,
nullptr, 0)) {
+ // Parse explicit offsets such as +08:00 with %Ez, without a separate
named-zone lookup.
+ success = cctz::parse("%Y-%m-%dT%H:%M:%E*S%Ez", text,
cctz::utc_time_zone(), &time);
+ } else {
+ // ES interprets an ISO timestamp without a suffix in UTC, not the
session time zone.
+ success = cctz::parse("%Y-%m-%dT%H:%M:%E*S", text,
cctz::utc_time_zone(), &time);
+ }
+ if (success) {
+ // A pre-epoch fractional instant floors to its containing second, as
numeric dates do.
+ *seconds =
std::chrono::floor<std::chrono::seconds>(time).time_since_epoch().count();
+ }
+ return success;
+}
+
+template <PrimitiveType T>
+static bool parse_es_calendar_date(const std::string& text, std::string_view
format,
+ typename PrimitiveTypeTraits<T>::CppType&
value,
+ const cctz::time_zone& time_zone) {
+ bool year_only;
+ if (!es_calendar_format_accepts(text, format, &year_only)) {
+ return false;
+ }
+ if (year_only) {
+ return parse_es_calendar_date<T>(text + "-01-01", format, value,
time_zone);
+ }
+ if (text.size() > 19 || (text.size() == 19 && text[10] == 'T')) {
+ int64_t seconds;
+ // Validate the civil year before narrowing it, just as epoch inputs
do.
+ return parse_es_iso_datetime(text, &seconds) &&
+ set_es_epoch_seconds(value, seconds, time_zone);
+ }
+ CastParameters params;
+ if constexpr (T == TYPE_DATE || T == TYPE_DATETIME) {
+ return
CastToDateOrDatetime::from_string_non_strict_mode<DatelikeTargetType::DATE_TIME>(
+ {text.data(), text.size()}, value, nullptr, params);
+ } else if constexpr (T == TYPE_DATEV2) {
+ return CastToDateV2::from_string_non_strict_mode({text.data(),
text.size()}, value, nullptr,
+ params);
+ } else {
+ return CastToDatetimeV2::from_string_non_strict_mode({text.data(),
text.size()}, value,
+ nullptr, -1,
params);
+ }
+}
+
+template <PrimitiveType T>
+static bool parse_es_date_string(const std::string& text, std::string_view
formats,
+ typename PrimitiveTypeTraits<T>::CppType&
value,
+ const cctz::time_zone& time_zone) {
+ if (formats.empty()) {
+ // Old FE requests carry no format context; preserve the default ES
date grammar.
+ formats = "strict_date_optional_time||epoch_millis";
+ }
+ while (!formats.empty()) {
+ const auto separator = formats.find("||");
+ const auto format = formats.substr(0, separator);
Review Comment:
[P2] Trim each mapped format alternative before dispatch
ES 6 accepts `epoch_millis ||yyyy-MM-dd`: [Joda trims each alternative
recursively](https://github.com/elastic/elasticsearch/blob/v6.8.23/server/src/main/java/org/elasticsearch/common/joda/Joda.java#L55-L60),
while retaining the combined pattern for [mapping
serialization](https://github.com/elastic/elasticsearch/blob/v6.8.23/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java#L468-L470).
FE likewise trims tokens for type inference, so this column is DATEV2, but
forwards the original format. Here the token is `epoch_millis `, so it misses
the epoch branch and falls through to the generic calendar parser. In a UTC
source scan, `"20240101"` therefore becomes `2024-01-01` instead of the correct
epoch date `1970-01-01`; ordinary 13-digit epoch strings such as
`"1677895728000"` fail altogether. These strings decoded as epochs before this
change. Trim separator-adjacent whitespace from each alternative, preserving
their order and the internal space in `yyyy-MM-dd HH
:mm:ss`, and add mapped-source coverage. This is independent of the
calendar-width finding and the earlier numeric-source thread: the defect is
failure to recognize an otherwise valid epoch grammar name.
##########
be/src/format/table/es/es_scroll_parser.cpp:
##########
@@ -427,103 +427,123 @@ static bool set_es_epoch_seconds(DateValue& value,
int64_t seconds,
return true;
}
-template <PrimitiveType T>
-Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type,
bool is_date_str,
- typename PrimitiveTypeTraits<T>::CppType* slot,
- const cctz::time_zone& time_zone) {
- constexpr bool is_datetime_v1 = T == TYPE_DATE || T == TYPE_DATETIME;
- typename PrimitiveTypeTraits<T>::CppType dt_val;
- if (is_date_str) {
- std::string str_date(col.GetString(), col.GetStringLength());
- size_t str_length = col.GetStringLength();
- // The default strict_date_optional_time parser wins over epoch_millis
for a four-digit year.
- // Thus "2024" means 2024-01-01, whereas "12345" means 12.345 seconds
after the epoch.
- if (str_length == 4 && str_date.find_first_not_of("0123456789") ==
std::string::npos) {
- str_date += "-01-01";
- str_length = str_date.size();
+static bool es_calendar_format_accepts(const std::string& text,
std::string_view format,
+ bool* year_only) {
+ const bool digits = !text.empty() && text.find_first_not_of("0123456789")
== std::string::npos;
+ *year_only = false;
+ // Compact dates win only for their mapped grammar. "20240101" is an epoch
in the default
+ // format, whereas "2024" is a year only when an optional-date format
precedes epoch_millis.
+ if (format == "basic_date" || format == "yyyyMMdd") {
+ return digits && text.size() == 8;
+ }
+ if (format == "strict_date_optional_time" || format ==
"date_optional_time" ||
+ format == "strict_date_optional_time_nanos") {
+ if (digits) {
+ *year_only = text.size() == 4;
+ return *year_only;
}
- bool success = false;
- int64_t epoch_seconds;
- // Source fallback also reaches short, zero and fractional epoch
strings, e.g. "0"/"1.5".
- if (parse_es_epoch_millis(str_date, &epoch_seconds)) {
- success = set_es_epoch_seconds(dt_val, epoch_seconds, time_zone);
- } else if (str_length > 19) {
- std::chrono::system_clock::time_point tp;
- // time_zone suffix pattern
- // Z/+08:00/-04:30
- RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)");
- bool ok = false;
- std::string fmt;
- if (time_zone_pattern.Match(str_date, 0, str_date.size(),
RE2::UNANCHORED, nullptr,
- 0)) {
- // with time_zone info
- // YYYY-MM-DDTHH:MM:SSZ or YYYY-MM-DDTHH:MM:SS+08:00
- // or 2022-08-08T12:10:10.000Z or YYYY-MM-DDTHH:MM:SS-08:00
- fmt = "%Y-%m-%dT%H:%M:%E*S%Ez";
- ok = cctz::parse(fmt, str_date, cctz::utc_time_zone(), &tp);
- } else {
- // without time_zone info
- // 2022-08-08T12:10:10.000
- fmt = "%Y-%m-%dT%H:%M:%E*S";
- // If the time without time_zone info, ES will assume it is
UTC time.
- // So we parse it in Doris with UTC time zone.
- ok = cctz::parse(fmt, str_date, cctz::utc_time_zone(), &tp);
- }
- if (ok) {
- // The local time zone can change by session variable
`time_zone`
- // We should use the user specified time zone, not the actual
system local time zone.
- success = true;
- // A pre-epoch fractional instant must floor to its containing
second, like numeric dates.
- dt_val.from_unixtime(
-
std::chrono::floor<std::chrono::seconds>(tp).time_since_epoch().count(),
- time_zone);
- }
- } else if (str_length == 19) {
- // YYYY-MM-DDTHH:MM:SS
- if (*(str_date.c_str() + 10) == 'T') {
- std::chrono::system_clock::time_point tp;
- const bool ok =
- cctz::parse("%Y-%m-%dT%H:%M:%S", str_date,
cctz::utc_time_zone(), &tp);
- if (ok) {
- success = true;
-
dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), time_zone);
- }
- } else {
- // YYYY-MM-DD HH:MM:SS
- CastParameters params;
- if constexpr (is_datetime_v1) {
- success =
CastToDateOrDatetime::from_string_non_strict_mode<
- DatelikeTargetType::DATE_TIME>({str_date.c_str(),
str_length}, dt_val,
- nullptr, params);
- } else if constexpr (T == TYPE_DATEV2) {
- success = CastToDateV2::from_string_non_strict_mode(
- {str_date.c_str(), str_length}, dt_val, nullptr,
params);
- } else {
- success = CastToDatetimeV2::from_string_non_strict_mode(
- {str_date.c_str(), str_length}, dt_val, nullptr,
-1, params);
- }
- }
+ return true;
+ }
+ // Restrict named calendar shapes before trying a later epoch alternative.
+ if (format == "yyyy-MM-dd" || format == "strict_date" || format == "date")
{
+ return text.size() == 10 && text[4] == '-' && text[7] == '-';
Review Comment:
[P2] Preserve variable-width dates in legacy source scans
On ES 6.8, a field mapped as `yyyy-MM-dd` can validly retain `"2024-1-1"` in
`_source`: [ES selects Joda for this
pattern](https://github.com/elastic/elasticsearch/blob/v6.8.23/server/src/main/java/org/elasticsearch/common/time/DateFormatter.java#L118-L124),
whose [numeric pattern widths do not require padding during
parsing](https://github.com/JodaOrg/joda-time/blob/v2.10.10/src/main/java/org/joda/time/format/DateTimeFormat.java#L93-L95).
FE infers DATEV2 and now forwards that format, so a source scan
(`enable_docvalue_scan=false`, or a source-mode fallback) rejects the
eight-character value here before reaching the non-strict cast that previously
read it successfully. The adjacent 19-character guard likewise rejects
`"2024-1-1 1:2:3"` under `yyyy-MM-dd HH:mm:ss`; selecting either field now
aborts the scan with a cast-format error. Preserve these legacy variable-width
calendar components while still excluding digit-only epoch text, and cover
scalar/date-array source reads. This i
s distinct from the existing ES 6 ISO-docvalue thread, which concerns the
response grammar of `fields`, not the original `_source` value.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]