This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a767472c56 [fix](DOE)Fix es p0 case error (#17502)
a767472c56 is described below
commit a767472c5653771db7ff838c7f50522bd6d9c0db
Author: qiye <[email protected]>
AuthorDate: Wed Mar 8 08:06:30 2023 +0800
[fix](DOE)Fix es p0 case error (#17502)
Fix es array parse error, introduced by #16806
---
be/src/exec/es/es_scroll_parser.cpp | 2 +
.../doris/external/elasticsearch/EsUtil.java | 46 ++++++++++++----------
.../doris/external/elasticsearch/EsUtilTest.java | 4 +-
3 files changed, 30 insertions(+), 22 deletions(-)
diff --git a/be/src/exec/es/es_scroll_parser.cpp
b/be/src/exec/es/es_scroll_parser.cpp
index e8e81c40f9..bf2f8f2732 100644
--- a/be/src/exec/es/es_scroll_parser.cpp
+++ b/be/src/exec/es/es_scroll_parser.cpp
@@ -519,6 +519,8 @@ Status ScrollParser::fill_columns(const TupleDescriptor*
tuple_desc,
const rapidjson::Value& str_col = is_nested_str ? col[0] : col;
+ RETURN_ERROR_IF_COL_IS_ARRAY(col, type);
+
const std::string& val = str_col.GetString();
size_t val_size = str_col.GetStringLength();
StringParser::ParseResult result;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
index 1357608eb2..f5ba8ffc38 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
@@ -101,20 +101,20 @@ public class EsUtil {
}
@VisibleForTesting
- public static ObjectNode getRootSchema(ObjectNode mappings, String
mappingType) {
+ public static ObjectNode getRootSchema(ObjectNode mappings, String
mappingType, List<String> arrayFields) {
// Type is null in the following three cases
// 1. Equal 6.8.x and after
// 2. Multi-catalog auto infer
// 3. Equal 6.8.x and before user not passed
if (mappingType == null) {
// remove dynamic templates, for ES 7.x and 8.x
- checkNonPropertiesFields(mappings);
+ checkNonPropertiesFields(mappings, arrayFields);
String firstType = mappings.fieldNames().next();
if (!"properties".equals(firstType)) {
// If type is not passed in takes the first type.
ObjectNode firstData = (ObjectNode) mappings.get(firstType);
// check for ES 6.x and before
- checkNonPropertiesFields(firstData);
+ checkNonPropertiesFields(firstData, arrayFields);
return firstData;
}
// Equal 7.x and after
@@ -123,11 +123,11 @@ public class EsUtil {
if (mappings.has(mappingType)) {
ObjectNode jsonData = (ObjectNode) mappings.get(mappingType);
// check for ES 6.x and before
- checkNonPropertiesFields(jsonData);
+ checkNonPropertiesFields(jsonData, arrayFields);
return jsonData;
}
// Compatible type error
- return getRootSchema(mappings, null);
+ return getRootSchema(mappings, null, arrayFields);
}
}
@@ -136,9 +136,21 @@ public class EsUtil {
*
* @param mappings
*/
- private static void checkNonPropertiesFields(ObjectNode mappings) {
- // remove `_meta` field
- mappings.remove("_meta");
+ private static void checkNonPropertiesFields(ObjectNode mappings,
List<String> arrayFields) {
+ // remove `_meta` field and parse array_fields
+ JsonNode metaNode = mappings.remove("_meta");
+ if (metaNode != null) {
+ JsonNode dorisMeta = metaNode.get("doris");
+ if (dorisMeta != null) {
+ JsonNode arrayNode = dorisMeta.get("array_fields");
+ if (arrayNode != null) {
+ Iterator<JsonNode> iterator = arrayNode.iterator();
+ while (iterator.hasNext()) {
+ arrayFields.add(iterator.next().asText());
+ }
+ }
+ }
+ }
// remove `dynamic_templates` field
mappings.remove("dynamic_templates");
// check explicit mapping
@@ -152,7 +164,7 @@ public class EsUtil {
**/
public static ObjectNode getMappingProps(String sourceIndex, String
indexMapping, String mappingType) {
ObjectNode mappings = getMapping(indexMapping);
- ObjectNode rootSchema = getRootSchema(mappings, mappingType);
+ ObjectNode rootSchema = getRootSchema(mappings, mappingType, new
ArrayList<>());
ObjectNode properties = (ObjectNode) rootSchema.get("properties");
if (properties == null) {
throw new DorisEsException(
@@ -169,13 +181,15 @@ public class EsUtil {
boolean mappingEsId) {
String mapping = client.getMapping(indexName);
ObjectNode mappings = getMapping(mapping);
- ObjectNode rootSchema = getRootSchema(mappings, mappingType);
- return genColumnsFromEs(indexName, mappingType, rootSchema,
mappingEsId);
+ // Get array_fields while removing _meta property.
+ List<String> arrayFields = new ArrayList<>();
+ ObjectNode rootSchema = getRootSchema(mappings, mappingType,
arrayFields);
+ return genColumnsFromEs(indexName, mappingType, rootSchema,
mappingEsId, arrayFields);
}
@VisibleForTesting
public static List<Column> genColumnsFromEs(String indexName, String
mappingType, ObjectNode rootSchema,
- boolean mappingEsId) {
+ boolean mappingEsId, List<String> arrayFields) {
List<Column> columns = new ArrayList<>();
if (mappingEsId) {
Column column = new Column();
@@ -191,14 +205,6 @@ public class EsUtil {
throw new DorisEsException(
"index[" + indexName + "] type[" + mappingType + "]
mapping not found for the ES Cluster");
}
- List<String> arrayFields = new ArrayList<>();
- JsonNode meta = mappingProps.get("_meta");
- if (meta != null) {
- JsonNode dorisMeta = meta.get("doris");
- if (dorisMeta != null) {
- arrayFields = dorisMeta.findValuesAsText("array_fields");
- }
- }
Iterator<String> iterator = mappingProps.fieldNames();
while (iterator.hasNext()) {
String fieldName = iterator.next();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
index 8816935ba5..6706ccd777 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
@@ -220,8 +220,8 @@ public class EsUtilTest extends EsTestCase {
@Test
public void testDateType() throws IOException, URISyntaxException {
ObjectNode testDateFormat = EsUtil.getRootSchema(
-
EsUtil.getMapping(loadJsonFromFile("data/es/test_date_format.json")), null);
- List<Column> parseColumns =
EsUtil.genColumnsFromEs("test_date_format", null, testDateFormat, false);
+
EsUtil.getMapping(loadJsonFromFile("data/es/test_date_format.json")), null, new
ArrayList<>());
+ List<Column> parseColumns =
EsUtil.genColumnsFromEs("test_date_format", null, testDateFormat, false, new
ArrayList<>());
Assertions.assertEquals(8, parseColumns.size());
for (Column column : parseColumns) {
String name = column.getName();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
