Gabriel39 commented on code in PR #70205:
URL: https://github.com/apache/doris/pull/70205#discussion_r4610949429


##########
regression-test/suites/external_table_p2/es/test_es_source_review_cases.groovy:
##########
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+
+suite("test_es_source_review_cases", "p2,external") {
+    if 
(!"true".equalsIgnoreCase(context.config.otherConfigs.get("enableEsTest"))) {
+        return
+    }
+    String host = context.config.otherConfigs.get("externalEnvIp")
+    String port = context.config.otherConfigs.get("es_8_port")
+    assertTrue(port != null && !port.isEmpty(), "es_8_port must be configured")
+    String endpoint = "http://${host}:${port}";
+    def request = { String method, String path, Object body = null ->
+        def connection = new URL("${endpoint}/${path}").openConnection()
+        connection.requestMethod = method
+        connection.connectTimeout = 10000
+        connection.readTimeout = 30000
+        try {
+            if (body != null) {
+                connection.doOutput = true
+                connection.setRequestProperty("Content-Type", 
"application/json")
+                connection.outputStream.withCloseable { 
it.write(JsonOutput.toJson(body).getBytes("UTF-8")) }
+            }
+            int status = connection.responseCode
+            if (method == "DELETE" && status == 404) {
+                return null
+            }
+            String text = (status < 400 ? connection.inputStream : 
connection.errorStream).getText("UTF-8")
+            assertTrue(status >= 200 && status < 300, "ES ${method} ${path}: 
${status} ${text}")
+            return new JsonSlurper().parseText(text)
+        } finally {
+            connection.disconnect()
+        }
+    }
+    request("DELETE", "doris_source_review_cases")
+    request("PUT", "doris_source_review_cases", [
+        settings: [number_of_shards: 1, number_of_replicas: 0,
+            analysis: [normalizer: [folded: [type: "custom", filter: 
["lowercase"]]]]],
+        mappings: [properties: [row_id: [type: "long"], tag: [type: "keyword", 
normalizer: "folded"],
+            n: [type: "integer"], f: [type: "float"], obj: [type: "flattened"],
+            epoch: [type: "date", format: "epoch_millis"], basic: [type: 
"date", format: "basic_date"]]]])
+    [[row_id: 1, tag: "UPPER", n: new BigDecimal("0.9999999999999999"),
+        f: new BigDecimal("1.0000000596046448"), obj: [:], epoch: "2024", 
basic: "20240101"],
+     [row_id: 2, tag: "UPPER", n: null, f: null, obj: null, epoch: 2024, 
basic: 20240101],
+     [row_id: 3, tag: "UPPER", n: 1, f: 1.0, obj: [key: "value"], epoch: "0", 
basic: "20240101"]].each { row ->
+        request("PUT", "doris_source_review_cases/_doc/${row.row_id}", row)
+    }
+    request("POST", "doris_source_review_cases/_refresh")
+
+    sql "switch internal"
+    sql "drop table if exists es_source_review_control"
+    sql """create table es_source_review_control (row_id bigint, tag string, n 
int, f float, obj json)
+        distributed by hash(row_id) buckets 1 
properties("replication_num"="1")"""
+    String control = "internal.${context.dbName}.es_source_review_control"
+    sql "drop catalog if exists es_source_review_catalog"
+    sql """create catalog es_source_review_catalog properties("type"="es", 
"hosts"="${endpoint}",
+        "nodes_discovery"="false", "enable_docvalue_scan"="true")"""
+    def originalScanner = sql("select @@enable_file_scanner_v2")[0][0]
+    def originalParallel = sql("show variables like 
'enable_es_parallel_scroll'")[0][1]
+    def originalZone = sql("select @@time_zone")[0][0]
+    try {
+        sql "set time_zone = 'UTC'"
+        sql "switch es_source_review_catalog"
+        // Compare filters with the values Doris actually materializes, 
independently of ES's
+        // numeric indexing conversion. The normalized tag makes this entire 
projection use source.
+        sql """insert into ${control} select row_id,tag,n,f,obj from 
doris_source_review_cases"""
+        [false, true].each { scanner ->
+            sql "set enable_file_scanner_v2 = ${scanner}"
+            [false, true].each { parallel ->
+                sql "set enable_es_parallel_scroll = ${parallel}"
+                String tag = "v2_${scanner}_parallel_${parallel}"
+                ["n = 1", "n != 1", "n > 0", "n is null", "n = 1 or row_id = 
3", "not (n = 1)",
+                 "f = cast(1 as float)", "f != cast(1 as float)", "obj is 
null", "obj is not null"].eachWithIndex {
+                    String predicate, int i ->
+                    quickTest("${tag}_filter_${i}",
+                        "select row_id,tag,n,f from doris_source_review_cases 
where ${predicate} order by row_id",
+                        "select row_id,tag,n,f from ${control} where 
${predicate} order by row_id")
+                }
+                quickTest("${tag}_local_filter_limit",
+                    "select row_id,tag from doris_source_review_cases where n 
= 1 order by row_id limit 1",
+                    "select row_id,tag from ${control} where n = 1 order by 
row_id limit 1")
+                // epoch_millis-only accepts both the string and numeric 2024 
as milliseconds, not a year.
+                quickTest("${tag}_epoch_format",
+                    "select row_id,tag,cast(epoch as date) from 
doris_source_review_cases order by row_id",

Review Comment:
   Already addressed in `1f479e100a6`: the decoder fixture uses 
`epoch_millis||yyyy-MM-dd`, is inferred as DATEV2 and is read directly. The 
separate `epoch_raw` field keeps the epoch-only BIGINT contract. A fresh 
recording-endpoint replay with the actual FE mapper confirms those two types, 
all three fixture rows and the eight epoch comparison pairs. This replay 
validates fixture construction, not live SQL execution.



##########
be/src/format/table/es/es_scroll_parser.cpp:
##########
@@ -564,36 +583,36 @@ Status get_date_value_int(const rapidjson::Value& col, 
PrimitiveType type, bool
 template <PrimitiveType T>
 Status get_date_int(const rapidjson::Value& col, PrimitiveType type, bool 
pure_doc_value,
                     typename PrimitiveTypeTraits<T>::CppType* slot,
-                    const cctz::time_zone& time_zone) {
-    // this would happend just only when `enable_docvalue_scan = false`, and 
field has timestamp format date from _source
+                    const cctz::time_zone& time_zone, std::string_view 
date_format) {
     if (col.IsNumber()) {
-        // ES process date/datetime field would use millisecond timestamp for 
index or docvalue
-        // processing date type field, if a number is encountered, Doris On ES 
will force it to be processed according to ms
-        // Doris On ES needs to be consistent with ES, so just divided by 1000 
because the unit for from_unixtime is seconds
-        return get_date_value_int<T>(col, type, false, slot, time_zone);
+        // ES applies the mapped grammar to source numeric tokens too. Numeric 
docvalues are
+        // canonical epoch values; requests from an old FE also retain their 
millisecond decoder.
+        const bool mapped_source = !pure_doc_value && !date_format.empty();

Review Comment:
   Rechecked; keeping the mapped source-number path. Elasticsearch's source 
parser obtains `textOrNull()` and passes that text to the field's mapped 
formatter, including numeric tokens; numeric query bounds and canonical numeric 
docvalues are different paths. See 
[DateFieldMapper.parseCreateField](https://github.com/elastic/elasticsearch/blob/v8.19.0/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java#L1090-L1101).
 `DateStringsHonorMappedFormatOrder` covers both quoted and unquoted inputs. 
The latest review also withdrew this finding. The compatibility fix in this 
update is restricted to ISO-shaped string docvalues and leaves this source 
behavior intact.



##########
fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsScanPlanProvider.java:
##########
@@ -299,12 +303,13 @@ private static void 
collectSourceUnsafeFields(ConnectorExpression expression, Se
         if (expression instanceof ConnectorColumnRef) {
             ConnectorColumnRef column = (ConnectorColumnRef) expression;
             String type = column.getType().getTypeName();
-            // Source arrays preserve their container (or JSON STRING), 
whereas ES matches elements
-            // and treats an empty array as missing. Source nullness must 
retain the local predicate.
+            // Source containers such as {} are nonnull even when ES indexes 
no leaf value.
+            // Source arrays also preserve their container (or JSON STRING), 
whereas ES matches elements.
             // _id comes from hit metadata and cannot acquire an array 
representation in source mode.
             if (!"_id".equals(column.getColumnName())
                     && ("STRING".equalsIgnoreCase(type) || 
"VARCHAR".equalsIgnoreCase(type)
-                            || "CHAR".equalsIgnoreCase(type) || 
"ARRAY".equalsIgnoreCase(type))) {
+                            || "CHAR".equalsIgnoreCase(type) || 
"ARRAY".equalsIgnoreCase(type)
+                            || "JSONB".equalsIgnoreCase(type))) {

Review Comment:
   Rechecked; the reported empty-object mismatch applies to source reads, where 
`sourceJsonbNullPredicatesRemainLocal` retains both null predicates locally. A 
pure docvalue request omits source; an absent `fields.obj` is materialized as 
SQL NULL, consistent with the ES exists predicate. Forcing that scan into 
source mode would change its materialized-value contract. This finding was also 
withdrawn in the [latest 
review](https://github.com/apache/doris/pull/70205#pullrequestreview-5939264876).
 The fresh ES connector suite passes all executed tests, including the source 
JSONB guard.



##########
be/src/format/table/es/es_scroll_parser.cpp:
##########
@@ -561,36 +581,36 @@ Status get_date_value_int(const rapidjson::Value& col, 
PrimitiveType type, bool
 template <PrimitiveType T>
 Status get_date_int(const rapidjson::Value& col, PrimitiveType type, bool 
pure_doc_value,
                     typename PrimitiveTypeTraits<T>::CppType* slot,
-                    const cctz::time_zone& time_zone) {
-    // this would happend just only when `enable_docvalue_scan = false`, and 
field has timestamp format date from _source
+                    const cctz::time_zone& time_zone, std::string_view 
date_format) {
     if (col.IsNumber()) {
-        // ES process date/datetime field would use millisecond timestamp for 
index or docvalue
-        // processing date type field, if a number is encountered, Doris On ES 
will force it to be processed according to ms
-        // Doris On ES needs to be consistent with ES, so just divided by 1000 
because the unit for from_unixtime is seconds
-        return get_date_value_int<T>(col, type, false, slot, time_zone);
+        // ES applies the mapped grammar to source numeric tokens too. Numeric 
docvalues are
+        // canonical epoch values; requests from an old FE also retain their 
millisecond decoder.
+        const bool mapped_source = !pure_doc_value && !date_format.empty();
+        return get_date_value_int<T>(col, type, mapped_source, slot, 
time_zone, date_format);
     } else if (col.IsArray() && pure_doc_value && !col.Empty()) {
         // this would happened just only when `enable_docvalue_scan = true`
         // ES add default format for all field after ES 6.4, if we not 
provided format for `date` field ES would impose
         // a standard date-format for date field as `2020-06-16T00:00:00.000Z`
         // At present, we just process this string format date. After some PR 
were merged into Doris, we would impose `epoch_mills` for
         // date field's docvalue
         if (col[0].IsString()) {
-            return get_date_value_int<T>(col[0], type, true, slot, time_zone);
+            return get_date_value_int<T>(col[0], type, true, slot, time_zone, 
date_format);

Review Comment:
   Fixed in `9011172c220b`. Scalar docvalue wrappers and already-unwrapped 
date-array elements now share normalization. ISO-shaped string docvalues use 
the legacy ISO response grammar; source values, non-ISO mapped strings and 
numeric docvalues retain their existing dispatch.
   
   Added five BE tests covering four scalar date types, date arrays, three 
timezones, modern/numeric controls and malformed inputs. Added an ES 6/8 
regression with source/docvalue and parallel-scroll comparisons.
   
   Verification: an extracted production routing/ISO probe reproduced 24 
failures before the fix and passes all 36 checks afterward under ASAN/UBSAN; 
its date-storage/calendar-cast adapters do not exercise full column 
materialization. Both affected C++ translation units pass syntax-only 
compilation; changed-line clang-tidy, clang-format 16 and header hygiene pass. 
The new Groovy suite compiles, and a recording-endpoint replay verifies real FE 
date-type inference and eight SQL comparison pairs. Standard BE unit execution 
stops at CMake generation because the installed dependencies lack MongoDB C 
driver headers; full BE and live ES regression execution remain for the newly 
requested CI run.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to