This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new fddef8b473 [fix](es-catalog)fix error when querying the index ,elasticsearch version 8.9.1 (#24839) fddef8b473 is described below commit fddef8b47340be920280e4f15688bdd32219be5f Author: Guangdong Liu <liug...@gmail.com> AuthorDate: Sun Oct 8 10:19:45 2023 +0800 [fix](es-catalog)fix error when querying the index ,elasticsearch version 8.9.1 (#24839) Issue Number: close #24833 --- .../elasticsearch/scripts/data/data5.json | 4 ++++ .../elasticsearch/scripts/es_init.sh | 9 +++++++++ .../elasticsearch/scripts/index/es7_test3.json | 22 ++++++++++++++++++++++ .../doris/external/elasticsearch/EsUtil.java | 14 ++++++++++---- .../data/external_table_p0/es/test_es_query.out | 5 +++++ .../external_table_p0/es/test_es_query.groovy | 3 ++- 6 files changed, 52 insertions(+), 5 deletions(-) diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data5.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data5.json new file mode 100755 index 0000000000..3d8852a528 --- /dev/null +++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data5.json @@ -0,0 +1,4 @@ +{ + "field1": "value1", + "field2": "value2" +} diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh b/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh index 8ddb125e32..2b396a95cf 100755 --- a/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh +++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh @@ -39,6 +39,8 @@ curl "http://${ES_6_HOST}:9200/test2_20220808/doc/_mapping" -H "Content-Type:app curl "http://${ES_7_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es7_test1.json" # create index test2_20220808 curl "http://${ES_7_HOST}:9200/test2_20220808" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json' +# create index test3_20231005 +curl "http://${ES_7_HOST}:9200/test3_20231005" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test3.json' # put data for tese1 curl "http://${ES_7_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json' curl "http://${ES_7_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' @@ -49,6 +51,8 @@ curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:applicati curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json' curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json' +# put data for test3_20231005 +curl "http://${ES_7_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json' # put _meta for array curl "http://${ES_7_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" @@ -59,6 +63,9 @@ curl "http://${ES_7_HOST}:9200/test2_20220808/_mapping" -H "Content-Type:applica curl "http://${ES_8_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es7_test1.json" # create index test2_20220808 curl "http://${ES_8_HOST}:9200/test2_20220808" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json' +# create index test3_20231005 +curl "http://${ES_8_HOST}:9200/test3_20231005" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test3.json' + # put data for tese1 curl "http://${ES_8_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json' curl "http://${ES_8_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' @@ -69,6 +76,8 @@ curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:applicati curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json' curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json' +# put data for test3_20231005 +curl "http://${ES_8_HOST}:9200/test3_20231005/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json' # put _meta for array curl "http://${ES_8_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test3.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test3.json new file mode 100644 index 0000000000..a8fa73b1b3 --- /dev/null +++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test3.json @@ -0,0 +1,22 @@ +{ + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings": { + "dynamic": "true", + "dynamic_date_formats": [ + "strict_date_optional_time", + "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z" + ], + "dynamic_templates": [ + + ], + "date_detection": true, + "numeric_detection": false, + "properties": { + "field1": { "type": "text" }, + "field2": { "type": "keyword" } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java index 2def928cce..021e92260b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java @@ -39,6 +39,9 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.StreamSupport; /** * Util for ES, some static method. @@ -110,15 +113,18 @@ public class EsUtil { // remove dynamic templates, for ES 7.x and 8.x checkNonPropertiesFields(mappings, arrayFields); String firstType = mappings.fieldNames().next(); - if (!"properties".equals(firstType)) { - // If type is not passed in takes the first type. + //The first parameter may not be properties, so we need to first determine whether it is 7.x or above. + if (StreamSupport.stream(Spliterators + .spliteratorUnknownSize(mappings.fieldNames(), Spliterator.ORDERED), false) + .anyMatch(s -> s.contains("properties"))) { + // Equal 7.x and after + return mappings; + } else { ObjectNode firstData = (ObjectNode) mappings.get(firstType); // check for ES 6.x and before checkNonPropertiesFields(firstData, arrayFields); return firstData; } - // Equal 7.x and after - return mappings; } else { if (mappings.has(mappingType)) { ObjectNode jsonData = (ObjectNode) mappings.get(mappingType); diff --git a/regression-test/data/external_table_p0/es/test_es_query.out b/regression-test/data/external_table_p0/es/test_es_query.out index b20cb439f4..fe033b5db3 100644 --- a/regression-test/data/external_table_p0/es/test_es_query.out +++ b/regression-test/data/external_table_p0/es/test_es_query.out @@ -96,6 +96,9 @@ true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0. [1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] +-- !sql710 -- +value1 value2 + -- !sql81 -- [1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"] ["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e", "f"] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 12345 @@ -132,3 +135,5 @@ true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0. [1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] [1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769, -32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"] ["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"] +-- !sql88 -- +value1 value2 diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy b/regression-test/suites/external_table_p0/es/test_es_query.groovy index 03b38cfc5f..23fc15c972 100644 --- a/regression-test/suites/external_table_p0/es/test_es_query.groovy +++ b/regression-test/suites/external_table_p0/es/test_es_query.groovy @@ -158,6 +158,7 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql77 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" order_qt_sql78 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1""" order_qt_sql79 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2""" + order_qt_sql710 """select * from test3_20231005""" sql """switch es8""" order_qt_sql81 """select * from test1 where test2='text#1'""" order_qt_sql82 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" @@ -166,6 +167,6 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") { order_qt_sql85 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" order_qt_sql86 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1""" order_qt_sql87 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2""" - + order_qt_sql88 """select * from test3_20231005""" } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org