github-actions[bot] commented on code in PR #70205:
URL: https://github.com/apache/doris/pull/70205#discussion_r4611087751


##########
be/src/format/table/es/es_scroll_parser.cpp:
##########
@@ -427,103 +427,123 @@ static bool set_es_epoch_seconds(DateValue& value, 
int64_t seconds,
     return true;
 }
 
-template <PrimitiveType T>
-Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, 
bool is_date_str,
-                          typename PrimitiveTypeTraits<T>::CppType* slot,
-                          const cctz::time_zone& time_zone) {
-    constexpr bool is_datetime_v1 = T == TYPE_DATE || T == TYPE_DATETIME;
-    typename PrimitiveTypeTraits<T>::CppType dt_val;
-    if (is_date_str) {
-        std::string str_date(col.GetString(), col.GetStringLength());
-        size_t str_length = col.GetStringLength();
-        // The default strict_date_optional_time parser wins over epoch_millis 
for a four-digit year.
-        // Thus "2024" means 2024-01-01, whereas "12345" means 12.345 seconds 
after the epoch.
-        if (str_length == 4 && str_date.find_first_not_of("0123456789") == 
std::string::npos) {
-            str_date += "-01-01";
-            str_length = str_date.size();
+static bool es_calendar_format_accepts(const std::string& text, 
std::string_view format,
+                                       bool* year_only) {
+    const bool digits = !text.empty() && text.find_first_not_of("0123456789") 
== std::string::npos;
+    *year_only = false;
+    // Compact dates win only for their mapped grammar. "20240101" is an epoch 
in the default
+    // format, whereas "2024" is a year only when an optional-date format 
precedes epoch_millis.
+    if (format == "basic_date" || format == "yyyyMMdd") {
+        return digits && text.size() == 8;
+    }
+    if (format == "strict_date_optional_time" || format == 
"date_optional_time" ||
+        format == "strict_date_optional_time_nanos") {
+        if (digits) {
+            *year_only = text.size() == 4;
+            return *year_only;
         }
-        bool success = false;
-        int64_t epoch_seconds;
-        // Source fallback also reaches short, zero and fractional epoch 
strings, e.g. "0"/"1.5".
-        if (parse_es_epoch_millis(str_date, &epoch_seconds)) {
-            success = set_es_epoch_seconds(dt_val, epoch_seconds, time_zone);
-        } else if (str_length > 19) {
-            std::chrono::system_clock::time_point tp;
-            // time_zone suffix pattern
-            // Z/+08:00/-04:30
-            RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)");
-            bool ok = false;
-            std::string fmt;
-            if (time_zone_pattern.Match(str_date, 0, str_date.size(), 
RE2::UNANCHORED, nullptr,
-                                        0)) {
-                // with time_zone info
-                // YYYY-MM-DDTHH:MM:SSZ or YYYY-MM-DDTHH:MM:SS+08:00
-                // or 2022-08-08T12:10:10.000Z or YYYY-MM-DDTHH:MM:SS-08:00
-                fmt = "%Y-%m-%dT%H:%M:%E*S%Ez";
-                ok = cctz::parse(fmt, str_date, cctz::utc_time_zone(), &tp);
-            } else {
-                // without time_zone info
-                // 2022-08-08T12:10:10.000
-                fmt = "%Y-%m-%dT%H:%M:%E*S";
-                // If the time without time_zone info, ES will assume it is 
UTC time.
-                // So we parse it in Doris with UTC time zone.
-                ok = cctz::parse(fmt, str_date, cctz::utc_time_zone(), &tp);
-            }
-            if (ok) {
-                // The local time zone can change by session variable 
`time_zone`
-                // We should use the user specified time zone, not the actual 
system local time zone.
-                success = true;
-                // A pre-epoch fractional instant must floor to its containing 
second, like numeric dates.
-                dt_val.from_unixtime(
-                        
std::chrono::floor<std::chrono::seconds>(tp).time_since_epoch().count(),
-                        time_zone);
-            }
-        } else if (str_length == 19) {
-            // YYYY-MM-DDTHH:MM:SS
-            if (*(str_date.c_str() + 10) == 'T') {
-                std::chrono::system_clock::time_point tp;
-                const bool ok =
-                        cctz::parse("%Y-%m-%dT%H:%M:%S", str_date, 
cctz::utc_time_zone(), &tp);
-                if (ok) {
-                    success = true;
-                    
dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), time_zone);
-                }
-            } else {
-                // YYYY-MM-DD HH:MM:SS
-                CastParameters params;
-                if constexpr (is_datetime_v1) {
-                    success = 
CastToDateOrDatetime::from_string_non_strict_mode<
-                            DatelikeTargetType::DATE_TIME>({str_date.c_str(), 
str_length}, dt_val,
-                                                           nullptr, params);
-                } else if constexpr (T == TYPE_DATEV2) {
-                    success = CastToDateV2::from_string_non_strict_mode(
-                            {str_date.c_str(), str_length}, dt_val, nullptr, 
params);
-                } else {
-                    success = CastToDatetimeV2::from_string_non_strict_mode(
-                            {str_date.c_str(), str_length}, dt_val, nullptr, 
-1, params);
-                }
-            }
+        return true;
+    }
+    // Restrict named calendar shapes before trying a later epoch alternative.
+    if (format == "yyyy-MM-dd" || format == "strict_date" || format == "date") 
{
+        return text.size() == 10 && text[4] == '-' && text[7] == '-';
+    }
+    if (format == "yyyy-MM-dd HH:mm:ss") {
+        return text.size() == 19 && text[4] == '-' && text[7] == '-' && 
text[10] == ' ';
+    }
+    return true;
+}
 
-        } else {
-            // YYYY-MM-DD or others
-            CastParameters params;
-            if constexpr (is_datetime_v1) {
-                success = CastToDateOrDatetime::from_string_non_strict_mode<
-                        DatelikeTargetType::DATE_TIME>({str_date.c_str(), 
str_length}, dt_val,
-                                                       nullptr, params);
-            } else if constexpr (T == TYPE_DATEV2) {
-                success = 
CastToDateV2::from_string_non_strict_mode({str_date.c_str(), str_length},
-                                                                    dt_val, 
nullptr, params);
-            } else {
-                success = CastToDatetimeV2::from_string_non_strict_mode(
-                        {str_date.c_str(), str_length}, dt_val, nullptr, -1, 
params);
+static bool parse_es_iso_datetime(const std::string& text, int64_t* seconds) {
+    std::chrono::system_clock::time_point time;
+    RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)");
+    bool success;
+    if (time_zone_pattern.Match(text, 0, text.size(), RE2::UNANCHORED, 
nullptr, 0)) {
+        // Parse explicit offsets such as +08:00 with %Ez, without a separate 
named-zone lookup.
+        success = cctz::parse("%Y-%m-%dT%H:%M:%E*S%Ez", text, 
cctz::utc_time_zone(), &time);
+    } else {
+        // ES interprets an ISO timestamp without a suffix in UTC, not the 
session time zone.
+        success = cctz::parse("%Y-%m-%dT%H:%M:%E*S", text, 
cctz::utc_time_zone(), &time);
+    }
+    if (success) {
+        // A pre-epoch fractional instant floors to its containing second, as 
numeric dates do.
+        *seconds = 
std::chrono::floor<std::chrono::seconds>(time).time_since_epoch().count();
+    }
+    return success;
+}
+
+template <PrimitiveType T>
+static bool parse_es_calendar_date(const std::string& text, std::string_view 
format,
+                                   typename PrimitiveTypeTraits<T>::CppType& 
value,
+                                   const cctz::time_zone& time_zone) {
+    bool year_only;
+    if (!es_calendar_format_accepts(text, format, &year_only)) {
+        return false;
+    }
+    if (year_only) {
+        return parse_es_calendar_date<T>(text + "-01-01", format, value, 
time_zone);
+    }
+    if (text.size() > 19 || (text.size() == 19 && text[10] == 'T')) {
+        int64_t seconds;
+        // Validate the civil year before narrowing it, just as epoch inputs 
do.
+        return parse_es_iso_datetime(text, &seconds) &&
+               set_es_epoch_seconds(value, seconds, time_zone);
+    }
+    CastParameters params;
+    if constexpr (T == TYPE_DATE || T == TYPE_DATETIME) {
+        return 
CastToDateOrDatetime::from_string_non_strict_mode<DatelikeTargetType::DATE_TIME>(
+                {text.data(), text.size()}, value, nullptr, params);
+    } else if constexpr (T == TYPE_DATEV2) {
+        return CastToDateV2::from_string_non_strict_mode({text.data(), 
text.size()}, value, nullptr,
+                                                         params);
+    } else {
+        return CastToDatetimeV2::from_string_non_strict_mode({text.data(), 
text.size()}, value,
+                                                             nullptr, -1, 
params);
+    }
+}
+
+template <PrimitiveType T>
+static bool parse_es_date_string(const std::string& text, std::string_view 
formats,
+                                 typename PrimitiveTypeTraits<T>::CppType& 
value,
+                                 const cctz::time_zone& time_zone) {
+    if (formats.empty()) {
+        // Old FE requests carry no format context; preserve the default ES 
date grammar.
+        formats = "strict_date_optional_time||epoch_millis";
+    }
+    while (!formats.empty()) {
+        const auto separator = formats.find("||");
+        const auto format = formats.substr(0, separator);

Review Comment:
   [P2] Trim each mapped format alternative before dispatch
   
   ES 6 accepts `epoch_millis ||yyyy-MM-dd`: [Joda trims each alternative 
recursively](https://github.com/elastic/elasticsearch/blob/v6.8.23/server/src/main/java/org/elasticsearch/common/joda/Joda.java#L55-L60),
 while retaining the combined pattern for [mapping 
serialization](https://github.com/elastic/elasticsearch/blob/v6.8.23/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java#L468-L470).
 FE likewise trims tokens for type inference, so this column is DATEV2, but 
forwards the original format. Here the token is `epoch_millis `, so it misses 
the epoch branch and falls through to the generic calendar parser. In a UTC 
source scan, `"20240101"` therefore becomes `2024-01-01` instead of the correct 
epoch date `1970-01-01`; ordinary 13-digit epoch strings such as 
`"1677895728000"` fail altogether. These strings decoded as epochs before this 
change. Trim separator-adjacent whitespace from each alternative, preserving 
their order and the internal space in `yyyy-MM-dd HH
 :mm:ss`, and add mapped-source coverage. This is independent of the 
calendar-width finding and the earlier numeric-source thread: the defect is 
failure to recognize an otherwise valid epoch grammar name.



##########
be/src/format/table/es/es_scroll_parser.cpp:
##########
@@ -427,103 +427,123 @@ static bool set_es_epoch_seconds(DateValue& value, 
int64_t seconds,
     return true;
 }
 
-template <PrimitiveType T>
-Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, 
bool is_date_str,
-                          typename PrimitiveTypeTraits<T>::CppType* slot,
-                          const cctz::time_zone& time_zone) {
-    constexpr bool is_datetime_v1 = T == TYPE_DATE || T == TYPE_DATETIME;
-    typename PrimitiveTypeTraits<T>::CppType dt_val;
-    if (is_date_str) {
-        std::string str_date(col.GetString(), col.GetStringLength());
-        size_t str_length = col.GetStringLength();
-        // The default strict_date_optional_time parser wins over epoch_millis 
for a four-digit year.
-        // Thus "2024" means 2024-01-01, whereas "12345" means 12.345 seconds 
after the epoch.
-        if (str_length == 4 && str_date.find_first_not_of("0123456789") == 
std::string::npos) {
-            str_date += "-01-01";
-            str_length = str_date.size();
+static bool es_calendar_format_accepts(const std::string& text, 
std::string_view format,
+                                       bool* year_only) {
+    const bool digits = !text.empty() && text.find_first_not_of("0123456789") 
== std::string::npos;
+    *year_only = false;
+    // Compact dates win only for their mapped grammar. "20240101" is an epoch 
in the default
+    // format, whereas "2024" is a year only when an optional-date format 
precedes epoch_millis.
+    if (format == "basic_date" || format == "yyyyMMdd") {
+        return digits && text.size() == 8;
+    }
+    if (format == "strict_date_optional_time" || format == 
"date_optional_time" ||
+        format == "strict_date_optional_time_nanos") {
+        if (digits) {
+            *year_only = text.size() == 4;
+            return *year_only;
         }
-        bool success = false;
-        int64_t epoch_seconds;
-        // Source fallback also reaches short, zero and fractional epoch 
strings, e.g. "0"/"1.5".
-        if (parse_es_epoch_millis(str_date, &epoch_seconds)) {
-            success = set_es_epoch_seconds(dt_val, epoch_seconds, time_zone);
-        } else if (str_length > 19) {
-            std::chrono::system_clock::time_point tp;
-            // time_zone suffix pattern
-            // Z/+08:00/-04:30
-            RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)");
-            bool ok = false;
-            std::string fmt;
-            if (time_zone_pattern.Match(str_date, 0, str_date.size(), 
RE2::UNANCHORED, nullptr,
-                                        0)) {
-                // with time_zone info
-                // YYYY-MM-DDTHH:MM:SSZ or YYYY-MM-DDTHH:MM:SS+08:00
-                // or 2022-08-08T12:10:10.000Z or YYYY-MM-DDTHH:MM:SS-08:00
-                fmt = "%Y-%m-%dT%H:%M:%E*S%Ez";
-                ok = cctz::parse(fmt, str_date, cctz::utc_time_zone(), &tp);
-            } else {
-                // without time_zone info
-                // 2022-08-08T12:10:10.000
-                fmt = "%Y-%m-%dT%H:%M:%E*S";
-                // If the time without time_zone info, ES will assume it is 
UTC time.
-                // So we parse it in Doris with UTC time zone.
-                ok = cctz::parse(fmt, str_date, cctz::utc_time_zone(), &tp);
-            }
-            if (ok) {
-                // The local time zone can change by session variable 
`time_zone`
-                // We should use the user specified time zone, not the actual 
system local time zone.
-                success = true;
-                // A pre-epoch fractional instant must floor to its containing 
second, like numeric dates.
-                dt_val.from_unixtime(
-                        
std::chrono::floor<std::chrono::seconds>(tp).time_since_epoch().count(),
-                        time_zone);
-            }
-        } else if (str_length == 19) {
-            // YYYY-MM-DDTHH:MM:SS
-            if (*(str_date.c_str() + 10) == 'T') {
-                std::chrono::system_clock::time_point tp;
-                const bool ok =
-                        cctz::parse("%Y-%m-%dT%H:%M:%S", str_date, 
cctz::utc_time_zone(), &tp);
-                if (ok) {
-                    success = true;
-                    
dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), time_zone);
-                }
-            } else {
-                // YYYY-MM-DD HH:MM:SS
-                CastParameters params;
-                if constexpr (is_datetime_v1) {
-                    success = 
CastToDateOrDatetime::from_string_non_strict_mode<
-                            DatelikeTargetType::DATE_TIME>({str_date.c_str(), 
str_length}, dt_val,
-                                                           nullptr, params);
-                } else if constexpr (T == TYPE_DATEV2) {
-                    success = CastToDateV2::from_string_non_strict_mode(
-                            {str_date.c_str(), str_length}, dt_val, nullptr, 
params);
-                } else {
-                    success = CastToDatetimeV2::from_string_non_strict_mode(
-                            {str_date.c_str(), str_length}, dt_val, nullptr, 
-1, params);
-                }
-            }
+        return true;
+    }
+    // Restrict named calendar shapes before trying a later epoch alternative.
+    if (format == "yyyy-MM-dd" || format == "strict_date" || format == "date") 
{
+        return text.size() == 10 && text[4] == '-' && text[7] == '-';

Review Comment:
   [P2] Preserve variable-width dates in legacy source scans
   
   On ES 6.8, a field mapped as `yyyy-MM-dd` can validly retain `"2024-1-1"` in 
`_source`: [ES selects Joda for this 
pattern](https://github.com/elastic/elasticsearch/blob/v6.8.23/server/src/main/java/org/elasticsearch/common/time/DateFormatter.java#L118-L124),
 whose [numeric pattern widths do not require padding during 
parsing](https://github.com/JodaOrg/joda-time/blob/v2.10.10/src/main/java/org/joda/time/format/DateTimeFormat.java#L93-L95).
 FE infers DATEV2 and now forwards that format, so a source scan 
(`enable_docvalue_scan=false`, or a source-mode fallback) rejects the 
eight-character value here before reaching the non-strict cast that previously 
read it successfully. The adjacent 19-character guard likewise rejects 
`"2024-1-1 1:2:3"` under `yyyy-MM-dd HH:mm:ss`; selecting either field now 
aborts the scan with a cast-format error. Preserve these legacy variable-width 
calendar components while still excluding digit-only epoch text, and cover 
scalar/date-array source reads. This i
 s distinct from the existing ES 6 ISO-docvalue thread, which concerns the 
response grammar of `fields`, not the original `_source` value.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to