This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3fccf206179 [fix](es) Fix array not supporting json subtype (#55738)
3fccf206179 is described below
commit 3fccf2061799bd10439de085e6bb390d02c36d27
Author: zy-kkk <[email protected]>
AuthorDate: Mon Sep 15 07:27:10 2025 +0800
[fix](es) Fix array not supporting json subtype (#55738)
An issue introduced by https://github.com/apache/doris/pull/40614, which
omitted handling of json subtypes when parsing single value for array
column.
---
be/src/exec/es/es_scroll_parser.cpp | 21 +++++++++++++++++++++
.../scripts/data/composite_type_array_bulk.json | 20 ++++++++++----------
.../index/array_meta_composite_type_array.json | 3 ++-
.../scripts/index/es6_composite_type_array.json | 10 +++++++++-
.../scripts/index/es7_composite_type_array.json | 10 +++++++++-
.../data/external_table_p0/es/test_es_query.out | Bin 154465 -> 162279 bytes
6 files changed, 51 insertions(+), 13 deletions(-)
diff --git a/be/src/exec/es/es_scroll_parser.cpp
b/be/src/exec/es/es_scroll_parser.cpp
index deb7aa140f1..94f8f0d1d56 100644
--- a/be/src/exec/es/es_scroll_parser.cpp
+++ b/be/src/exec/es/es_scroll_parser.cpp
@@ -527,6 +527,24 @@ Status process_date_column(const rapidjson::Value& col,
PrimitiveType sub_type,
return Status::OK();
}
+Status process_jsonb_column(const rapidjson::Value& col, PrimitiveType
sub_type,
+ bool pure_doc_value, vectorized::Array& array) {
+ if (!col.IsArray()) {
+ JsonBinaryValue jsonb_value;
+
RETURN_IF_ERROR(jsonb_value.from_json_string(json_value_to_string(col)));
+ vectorized::JsonbField json(jsonb_value.value(), jsonb_value.size());
+ array.push_back(vectorized::Field::create_field<TYPE_JSONB>(json));
+ } else {
+ for (const auto& sub_col : col.GetArray()) {
+ JsonBinaryValue jsonb_value;
+
RETURN_IF_ERROR(jsonb_value.from_json_string(json_value_to_string(sub_col)));
+ vectorized::JsonbField json(jsonb_value.value(),
jsonb_value.size());
+ array.push_back(vectorized::Field::create_field<TYPE_JSONB>(json));
+ }
+ }
+ return Status::OK();
+}
+
Status ScrollParser::parse_column(const rapidjson::Value& col, PrimitiveType
sub_type,
bool pure_doc_value, vectorized::Array&
array,
const cctz::time_zone& time_zone) {
@@ -561,6 +579,9 @@ Status ScrollParser::parse_column(const rapidjson::Value&
col, PrimitiveType sub
return process_date_column<TYPE_DATETIMEV2>(col, sub_type,
pure_doc_value, array,
time_zone);
}
+ case TYPE_JSONB: {
+ return process_jsonb_column(col, sub_type, pure_doc_value, array);
+ }
default:
LOG(ERROR) << "Do not support Array type: " << sub_type;
return Status::InternalError("Unsupported type");
diff --git
a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/composite_type_array_bulk.json
b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/composite_type_array_bulk.json
index 6d84244316a..5bb3bf67f2a 100755
---
a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/composite_type_array_bulk.json
+++
b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/composite_type_array_bulk.json
@@ -1,10 +1,10 @@
-{"name": "Andy", "sports": "soccer", "scores": 100}
-{"name": "Betty", "sports": "pingpong ball", "scores": 90}
-{"name": "Cindy", "sports": "武术", "scores": 89}
-{"name": "David", "sports": ["volleyball"], "scores": [77]}
-{"name": "Emily", "sports": ["baseball", "golf", "hockey"], "scores": [56, 78,
99]}
-{"name": "Frank", "sports": ["rugby", "cricket", "boxing"], "scores": [45, 67,
88]}
-{"name": "Grace", "sports": ["table tennis", "badminton", "athletics"],
"scores": [34, 56, 78]}
-{"name": "Henry", "sports": ["archery", "fencing", "weightlifting"], "scores":
[23, 45, 67]}
-{"name": "Ivy", "sports": ["judo", "karate", "taekwondo"], "scores": [12, 34,
56]}
-{"name": "Jack", "sports": ["wrestling", "gymnastics", "surfing"], "scores":
[1, 23, 45]}
\ No newline at end of file
+{"name": "Andy", "sports": "soccer", "scores": 100, "z_details": [{"position":
"forward", "club": "Manchester", "salary": 50000}]}
+{"name": "Betty", "sports": "pingpong ball", "scores": 90, "z_details":
[{"position": "attacker", "club": "Beijing", "salary": 30000}]}
+{"name": "Cindy", "sports": "武术", "scores": 89, "z_details": [{"position":
"fighter", "club": "Shaolin", "salary": 25000}]}
+{"name": "David", "sports": ["volleyball"], "scores": [77], "z_details":
[{"position": "spiker", "club": "Tokyo", "salary": 40000}]}
+{"name": "Emily", "sports": ["baseball", "golf", "hockey"], "scores": [56, 78,
99], "z_details": [{"position": "pitcher", "club": "Yankees", "salary": 80000},
{"position": "driver", "club": "PGA", "salary": 120000}, {"position": "center",
"club": "Rangers", "salary": 90000}]}
+{"name": "Frank", "sports": ["rugby", "cricket", "boxing"], "scores": [45, 67,
88], "z_details": [{"position": "scrum-half", "club": "Leicester", "salary":
60000}, {"position": "batsman", "club": "Mumbai", "salary": 70000},
{"position": "heavyweight", "club": "Vegas", "salary": 150000}]}
+{"name": "Grace", "sports": ["table tennis", "badminton", "athletics"],
"scores": [34, 56, 78], "z_details": [{"position": "attacker", "club": "China",
"salary": 35000}, {"position": "singles", "club": "Denmark", "salary": 45000},
{"position": "sprinter", "club": "Jamaica", "salary": 85000}]}
+{"name": "Henry", "sports": ["archery", "fencing", "weightlifting"], "scores":
[23, 45, 67], "z_details": [{"position": "recurve", "club": "Korea", "salary":
20000}, {"position": "epee", "club": "France", "salary": 30000}, {"position":
"heavyweight", "club": "Bulgaria", "salary": 55000}]}
+{"name": "Ivy", "sports": ["judo", "karate", "taekwondo"], "scores": [12, 34,
56], "z_details": [{"position": "lightweight", "club": "Japan", "salary":
25000}, {"position": "kata", "club": "Okinawa", "salary": 22000}, {"position":
"middleweight", "club": "Korea", "salary": 28000}]}
+{"name": "Jack", "sports": ["wrestling", "gymnastics", "surfing"], "scores":
[1, 23, 45], "z_details": [{"position": "freestyle", "club": "Russia",
"salary": 15000}, {"position": "rings", "club": "Romania", "salary": 35000},
{"position": "longboard", "club": "Hawaii", "salary": 50000}]}
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/array_meta_composite_type_array.json
b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/array_meta_composite_type_array.json
index a9ccc1ab850..6adb86672da 100644
---
a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/array_meta_composite_type_array.json
+++
b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/array_meta_composite_type_array.json
@@ -3,7 +3,8 @@
"doris":{
"array_fields":[
"sports",
- "scores"
+ "scores",
+ "z_details"
]
}
}
diff --git
a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_composite_type_array.json
b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_composite_type_array.json
index 3a38a8f22d3..081be7fc166 100755
---
a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_composite_type_array.json
+++
b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_composite_type_array.json
@@ -8,7 +8,15 @@
"properties": {
"name": { "type": "keyword" },
"sports": { "type": "keyword", "doc_values": false},
- "scores": { "type": "integer", "doc_values": false}
+ "scores": { "type": "integer", "doc_values": false},
+ "z_details": {
+ "type": "object",
+ "properties": {
+ "position": { "type": "keyword" },
+ "club": { "type": "keyword" },
+ "salary": { "type": "integer" }
+ }
+ }
}
}
}
diff --git
a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_composite_type_array.json
b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_composite_type_array.json
index 3529cc61eb7..6af71903ca7 100644
---
a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_composite_type_array.json
+++
b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_composite_type_array.json
@@ -7,7 +7,15 @@
"properties": {
"name": { "type": "keyword" },
"sports": { "type": "keyword", "doc_values": false},
- "scores": { "type": "integer", "doc_values": false}
+ "scores": { "type": "integer", "doc_values": false},
+ "z_details": {
+ "type": "object",
+ "properties": {
+ "position": { "type": "keyword" },
+ "club": { "type": "keyword" },
+ "salary": { "type": "integer" }
+ }
+ }
}
}
}
diff --git a/regression-test/data/external_table_p0/es/test_es_query.out
b/regression-test/data/external_table_p0/es/test_es_query.out
index 70cc0f4b04a..e517648c949 100644
Binary files a/regression-test/data/external_table_p0/es/test_es_query.out and
b/regression-test/data/external_table_p0/es/test_es_query.out differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
