This is an automated email from the ASF dual-hosted git repository.
stalary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 87439e227e [Enhancement](DOE): Doe support object/nested use string
(#12401)
87439e227e is described below
commit 87439e227e02ad1903b9c926e5eb5d2ff3e42eed
Author: Stalary <[email protected]>
AuthorDate: Tue Sep 13 09:59:48 2022 +0800
[Enhancement](DOE): Doe support object/nested use string (#12401)
* MOD: doe support object/nested use string
---
be/src/http/http_client.cpp | 4 ++-
.../doris/external/elasticsearch/EsUtil.java | 37 +++++++++++-----------
.../doris/external/elasticsearch/MappingPhase.java | 3 ++
.../doris/external/elasticsearch/EsUtilTest.java | 8 +++++
.../test/resources/data/es/test_index_mapping.json | 30 ++++++++++++++++++
.../data/es/test_index_mapping_after_7x.json | 30 ++++++++++++++++++
.../es/test_index_mapping_field_mult_analyzer.json | 30 ++++++++++++++++++
7 files changed, 123 insertions(+), 19 deletions(-)
diff --git a/be/src/http/http_client.cpp b/be/src/http/http_client.cpp
index 96d9c447e4..0cb5b97ec4 100644
--- a/be/src/http/http_client.cpp
+++ b/be/src/http/http_client.cpp
@@ -18,6 +18,7 @@
#include "http/http_client.h"
#include "common/config.h"
+#include "util/stack_util.h"
namespace doris {
@@ -162,7 +163,8 @@ Status HttpClient::execute(const std::function<bool(const
void* data, size_t len
_callback = &callback;
auto code = curl_easy_perform(_curl);
if (code != CURLE_OK) {
- LOG(WARNING) << "fail to execute HTTP client, errmsg=" <<
_to_errmsg(code);
+ LOG(WARNING) << "fail to execute HTTP client, errmsg=" <<
_to_errmsg(code)
+ << ", trace=" << get_stack_trace();
return Status::InternalError(_to_errmsg(code));
}
return Status::OK();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
index 33dd1ce97d..c9b89e8f5d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
@@ -37,6 +37,7 @@ import org.apache.doris.analysis.RangePartitionDesc;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
@@ -358,22 +359,23 @@ public class EsUtil {
List<Column> columns = new ArrayList<>();
for (String key : keys) {
JSONObject field = (JSONObject) mappingProps.get(key);
- // Complex types are not currently supported.
+ Type type;
+ // Complex types are treating as String types for now.
if (field.containsKey("type")) {
- Type type = toDorisType(field.get("type").toString());
- if (!type.isInvalid()) {
- Column column = new Column();
- column.setName(key);
- column.setIsKey(true);
- column.setIsAllowNull(true);
- if (arrayFields.contains(key)) {
- column.setType(ArrayType.create(type, true));
- } else {
- column.setType(type);
- }
- columns.add(column);
- }
+ type = toDorisType(field.get("type").toString());
+ } else {
+ type = Type.STRING;
+ }
+ Column column = new Column();
+ column.setName(key);
+ column.setIsKey(true);
+ column.setIsAllowNull(true);
+ if (arrayFields.contains(key)) {
+ column.setType(ArrayType.create(type, true));
+ } else {
+ column.setType(type);
}
+ columns.add(column);
}
return columns;
}
@@ -403,16 +405,15 @@ public class EsUtil {
case "double":
case "scaled_float":
return Type.DOUBLE;
+ case "date":
+ return ScalarType.getDefaultDateType(Type.DATE);
case "keyword":
case "text":
case "ip":
case "nested":
case "object":
- return Type.STRING;
- case "date":
- return Type.DATE;
default:
- return Type.INVALID;
+ return Type.STRING;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
index d0082c45e8..493fa413a3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
@@ -119,6 +119,9 @@ public class MappingPhase implements SearchPhase {
if (!docValue) {
return;
}
+ } else if (fieldType == null || "nested".equals(fieldType)) {
+ // The object field has no type, and nested not support doc
value.
+ return;
}
docValueField = colName;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
index 9a00b58b4d..2e53e5f1ae 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
@@ -74,9 +74,15 @@ public class EsUtilTest extends EsTestCase {
Column k1 = new Column("k1", PrimitiveType.BIGINT);
Column k2 = new Column("k2", PrimitiveType.VARCHAR);
Column k3 = new Column("k3", PrimitiveType.VARCHAR);
+ Column k4 = new Column("k4", PrimitiveType.VARCHAR);
+ Column k5 = new Column("k5", PrimitiveType.VARCHAR);
+ Column k6 = new Column("k6", PrimitiveType.DATE);
columns.add(k1);
columns.add(k2);
columns.add(k3);
+ columns.add(k4);
+ columns.add(k5);
+ columns.add(k6);
}
@Test
@@ -120,6 +126,8 @@ public class EsUtilTest extends EsTestCase {
Assertions.assertEquals("k3.keyword",
searchContext1.docValueFieldsContext().get("k3"));
Assertions.assertEquals("k1",
searchContext1.docValueFieldsContext().get("k1"));
Assertions.assertEquals("k2",
searchContext1.docValueFieldsContext().get("k2"));
+
Assertions.assertNull(searchContext1.docValueFieldsContext().get("k4"));
+
Assertions.assertNull(searchContext1.docValueFieldsContext().get("k5"));
}
diff --git a/fe/fe-core/src/test/resources/data/es/test_index_mapping.json
b/fe/fe-core/src/test/resources/data/es/test_index_mapping.json
index 297a7fbb9d..d5e12bc155 100644
--- a/fe/fe-core/src/test/resources/data/es/test_index_mapping.json
+++ b/fe/fe-core/src/test/resources/data/es/test_index_mapping.json
@@ -16,6 +16,36 @@
"type": "keyword"
}
}
+ },
+ "k4": {
+ "properties": {
+ "child1": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "child2": {
+ "type": "long"
+ }
+ }
+ },
+ "k5": {
+ "type": "nested",
+ "properties": {
+ "child3": {
+ "type": "keyword"
+ },
+ "child4": {
+ "type": "keyword"
+ }
+ }
+ },
+ "k6": {
+ "type": "date"
}
}
}
diff --git
a/fe/fe-core/src/test/resources/data/es/test_index_mapping_after_7x.json
b/fe/fe-core/src/test/resources/data/es/test_index_mapping_after_7x.json
index d018360c48..c4a8e8528b 100644
--- a/fe/fe-core/src/test/resources/data/es/test_index_mapping_after_7x.json
+++ b/fe/fe-core/src/test/resources/data/es/test_index_mapping_after_7x.json
@@ -15,6 +15,36 @@
"type": "keyword"
}
}
+ },
+ "k4": {
+ "properties": {
+ "child1": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "child2": {
+ "type": "long"
+ }
+ }
+ },
+ "k5": {
+ "type": "nested",
+ "properties": {
+ "child3": {
+ "type": "keyword"
+ },
+ "child4": {
+ "type": "keyword"
+ }
+ }
+ },
+ "k6": {
+ "type": "date"
}
}
}
diff --git
a/fe/fe-core/src/test/resources/data/es/test_index_mapping_field_mult_analyzer.json
b/fe/fe-core/src/test/resources/data/es/test_index_mapping_field_mult_analyzer.json
index 1a002bd2db..5f3d785b39 100644
---
a/fe/fe-core/src/test/resources/data/es/test_index_mapping_field_mult_analyzer.json
+++
b/fe/fe-core/src/test/resources/data/es/test_index_mapping_field_mult_analyzer.json
@@ -16,6 +16,36 @@
"analyzer": "ik_max_word"
}
}
+ },
+ "k4": {
+ "properties": {
+ "child1": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "child2": {
+ "type": "long"
+ }
+ }
+ },
+ "k5": {
+ "type": "nested",
+ "properties": {
+ "child3": {
+ "type": "keyword"
+ },
+ "child4": {
+ "type": "keyword"
+ }
+ }
+ },
+ "k6": {
+ "type": "date"
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
