This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fddef8b473 [fix](es-catalog)fix error when querying the index
,elasticsearch version 8.9.1 (#24839)
fddef8b473 is described below
commit fddef8b47340be920280e4f15688bdd32219be5f
Author: Guangdong Liu <[email protected]>
AuthorDate: Sun Oct 8 10:19:45 2023 +0800
[fix](es-catalog)fix error when querying the index ,elasticsearch version
8.9.1 (#24839)
Issue Number: close #24833
---
.../elasticsearch/scripts/data/data5.json | 4 ++++
.../elasticsearch/scripts/es_init.sh | 9 +++++++++
.../elasticsearch/scripts/index/es7_test3.json | 22 ++++++++++++++++++++++
.../doris/external/elasticsearch/EsUtil.java | 14 ++++++++++----
.../data/external_table_p0/es/test_es_query.out | 5 +++++
.../external_table_p0/es/test_es_query.groovy | 3 ++-
6 files changed, 52 insertions(+), 5 deletions(-)
diff --git
a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data5.json
b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data5.json
new file mode 100755
index 0000000000..3d8852a528
--- /dev/null
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data5.json
@@ -0,0 +1,4 @@
+{
+ "field1": "value1",
+ "field2": "value2"
+}
diff --git
a/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh
b/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh
index 8ddb125e32..2b396a95cf 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh
@@ -39,6 +39,8 @@ curl "http://${ES_6_HOST}:9200/test2_20220808/doc/_mapping";
-H "Content-Type:app
curl "http://${ES_7_HOST}:9200/test1"; -H "Content-Type:application/json" -X
PUT -d "@/mnt/scripts/index/es7_test1.json"
# create index test2_20220808
curl "http://${ES_7_HOST}:9200/test2_20220808"; -H
"Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json'
+# create index test3_20231005
+curl "http://${ES_7_HOST}:9200/test3_20231005"; -H
"Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test3.json'
# put data for tese1
curl "http://${ES_7_HOST}:9200/test1/_doc/1"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
curl "http://${ES_7_HOST}:9200/test1/_doc/2"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
@@ -49,6 +51,8 @@ curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/1"; -H
"Content-Type:applicati
curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/2"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/3"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/4"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
+# put data for test3_20231005
+curl "http://${ES_7_HOST}:9200/test3_20231005/_doc/1"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
# put _meta for array
curl "http://${ES_7_HOST}:9200/test1/_mapping"; -H
"Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"
@@ -59,6 +63,9 @@ curl "http://${ES_7_HOST}:9200/test2_20220808/_mapping"; -H
"Content-Type:applica
curl "http://${ES_8_HOST}:9200/test1"; -H "Content-Type:application/json" -X
PUT -d "@/mnt/scripts/index/es7_test1.json"
# create index test2_20220808
curl "http://${ES_8_HOST}:9200/test2_20220808"; -H
"Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json'
+# create index test3_20231005
+curl "http://${ES_8_HOST}:9200/test3_20231005"; -H
"Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test3.json'
+
# put data for tese1
curl "http://${ES_8_HOST}:9200/test1/_doc/1"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
curl "http://${ES_8_HOST}:9200/test1/_doc/2"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
@@ -69,6 +76,8 @@ curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/1"; -H
"Content-Type:applicati
curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/2"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/3"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/4"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
+# put data for test3_20231005
+curl "http://${ES_8_HOST}:9200/test3_20231005/_doc/1"; -H
"Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data5.json'
# put _meta for array
curl "http://${ES_8_HOST}:9200/test1/_mapping"; -H
"Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json"
diff --git
a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test3.json
b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test3.json
new file mode 100644
index 0000000000..a8fa73b1b3
--- /dev/null
+++
b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test3.json
@@ -0,0 +1,22 @@
+{
+ "settings": {
+ "number_of_shards": 1,
+ "number_of_replicas": 0
+ },
+ "mappings": {
+ "dynamic": "true",
+ "dynamic_date_formats": [
+ "strict_date_optional_time",
+ "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
+ ],
+ "dynamic_templates": [
+
+ ],
+ "date_detection": true,
+ "numeric_detection": false,
+ "properties": {
+ "field1": { "type": "text" },
+ "field2": { "type": "keyword" }
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
index 2def928cce..021e92260b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
@@ -39,6 +39,9 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.StreamSupport;
/**
* Util for ES, some static method.
@@ -110,15 +113,18 @@ public class EsUtil {
// remove dynamic templates, for ES 7.x and 8.x
checkNonPropertiesFields(mappings, arrayFields);
String firstType = mappings.fieldNames().next();
- if (!"properties".equals(firstType)) {
- // If type is not passed in takes the first type.
+ //The first parameter may not be properties, so we need to first
determine whether it is 7.x or above.
+ if (StreamSupport.stream(Spliterators
+ .spliteratorUnknownSize(mappings.fieldNames(),
Spliterator.ORDERED), false)
+ .anyMatch(s -> s.contains("properties"))) {
+ // Equal 7.x and after
+ return mappings;
+ } else {
ObjectNode firstData = (ObjectNode) mappings.get(firstType);
// check for ES 6.x and before
checkNonPropertiesFields(firstData, arrayFields);
return firstData;
}
- // Equal 7.x and after
- return mappings;
} else {
if (mappings.has(mappingType)) {
ObjectNode jsonData = (ObjectNode) mappings.get(mappingType);
diff --git a/regression-test/data/external_table_p0/es/test_es_query.out
b/regression-test/data/external_table_p0/es/test_es_query.out
index b20cb439f4..fe033b5db3 100644
--- a/regression-test/data/external_table_p0/es/test_es_query.out
+++ b/regression-test/data/external_table_p0/es/test_es_query.out
@@ -96,6 +96,9 @@ true 1 128 32768 -1 0 1.0 1.0
1.0 1.0 2020-01-01 2020-01-01T12:00 a d
192.168.0.
[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769,
-32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4]
[1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01
12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"]
["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}",
"{"name":"Tim","age":28}"]
[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769,
-32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4]
[1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01
12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"]
["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}",
"{"name":"Tim","age":28}"]
+-- !sql710 --
+value1 value2
+
-- !sql81 --
[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01
12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1,
2, 3, 4] [32768, 32769, -32769, -32770] ["192.168.0.1", "127.0.0.1"]
["a", "b", "c"] [-1, 0, 1, 2] ["{"name":"Andy","age":18}",
"{"name":"Tim","age":28}"] [1, 2, 3, 4] [128, 129, -129, -130] ["d", "e",
"f"] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00
2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10
2022-08-08T20:10:10 12345
@@ -132,3 +135,5 @@ true 1 128 32768 -1 0 1.0
1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d
192.168.0.
[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769,
-32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4]
[1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01
12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"]
["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}",
"{"name":"Tim","age":28}"]
[1, 0, 1, 1] [1, -2, -3, 4] [128, 129, -129, -130] [32768, 32769, -32769,
-32770] [-1, 0, 1, 2] [0, 1, 2, 3] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4]
[1, 2, 3, 4] [1, 2, 3, 4] [2020-01-01, 2020-01-02] [2020-01-01
12:00:00, 2020-01-02 13:01:01] ["a", "b", "c"] ["d", "e", "f"]
["192.168.0.1", "127.0.0.1"] ["{"name":"Andy","age":18}",
"{"name":"Tim","age":28}"]
+-- !sql88 --
+value1 value2
diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy
b/regression-test/suites/external_table_p0/es/test_es_query.groovy
index 03b38cfc5f..23fc15c972 100644
--- a/regression-test/suites/external_table_p0/es/test_es_query.groovy
+++ b/regression-test/suites/external_table_p0/es/test_es_query.groovy
@@ -158,6 +158,7 @@ suite("test_es_query",
"p0,external,es,external_docker,external_docker_es") {
order_qt_sql77 """select * from test1 where esquery(test2,
'{"match":{"test2":"text#1"}}')"""
order_qt_sql78 """select c_bool, c_byte, c_short, c_integer, c_long,
c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date,
c_datetime, c_keyword, c_text, c_ip, c_person from test1"""
order_qt_sql79 """select c_bool, c_byte, c_short, c_integer, c_long,
c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date,
c_datetime, c_keyword, c_text, c_ip, c_person from test2"""
+ order_qt_sql710 """select * from test3_20231005"""
sql """switch es8"""
order_qt_sql81 """select * from test1 where test2='text#1'"""
order_qt_sql82 """select * from test2_20220808 where test4 >=
'2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'"""
@@ -166,6 +167,6 @@ suite("test_es_query",
"p0,external,es,external_docker,external_docker_es") {
order_qt_sql85 """select * from test1 where esquery(test2,
'{"match":{"test2":"text#1"}}')"""
order_qt_sql86 """select c_bool, c_byte, c_short, c_integer, c_long,
c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date,
c_datetime, c_keyword, c_text, c_ip, c_person from test1"""
order_qt_sql87 """select c_bool, c_byte, c_short, c_integer, c_long,
c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date,
c_datetime, c_keyword, c_text, c_ip, c_person from test2"""
-
+ order_qt_sql88 """select * from test3_20231005"""
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
