This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 4c793c25cc [fix](es catalog) remove _default_ field when parsing
mapping. (#24781)
4c793c25cc is described below
commit 4c793c25ccb7eb58db6fab86dd45f81d989650e4
Author: qiye <[email protected]>
AuthorDate: Sat Sep 23 10:58:55 2023 +0800
[fix](es catalog) remove _default_ field when parsing mapping. (#24781)
---
.../doris/external/elasticsearch/EsUtil.java | 4 +
.../doris/external/elasticsearch/EsUtilTest.java | 7 +
.../test/resources/data/es/default_mappings.json | 149 +++++++++++++++++++++
3 files changed, 160 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
index 31f8840432..2def928cce 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
@@ -154,6 +154,10 @@ public class EsUtil {
mappings.remove("dynamic_templates");
// remove `dynamic` field
mappings.remove("dynamic");
+ // remove `_default` field, we do not parse `_default_` mapping, only
explicit mapping.
+ // `_default` _mapping type is deprecated in 7.0 and removed in 8.0
+ //
https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html
+ mappings.remove("_default_");
// check explicit mapping
if (mappings.isEmpty()) {
throw new DorisEsException("Do not support index without explicit
mapping.");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
index 4fce17cc1a..1cdf1cb94d 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
@@ -278,5 +278,12 @@ public class EsUtilTest extends EsTestCase {
}
+ @Test
+ public void testDefaultMapping() throws IOException, URISyntaxException {
+ ObjectNode testAliases = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/default_mappings.json"),
+ null);
+
Assertions.assertEquals("{\"@timestamp\":{\"type\":\"date\"},\"@version\":{\"type\":\"keyword\"},\"act\":{\"type\":\"text\",\"norms\":false,\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"app\":{\"type\":\"text\",\"norms\":false,\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"tags\":{\"type\":\"text\",\"norms\":false,\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}},\"type\":{\"type\":\"text\",\"norms\":false,\"f
[...]
+ testAliases.toString());
+ }
}
diff --git a/fe/fe-core/src/test/resources/data/es/default_mappings.json
b/fe/fe-core/src/test/resources/data/es/default_mappings.json
new file mode 100644
index 0000000000..fd28dcffe5
--- /dev/null
+++ b/fe/fe-core/src/test/resources/data/es/default_mappings.json
@@ -0,0 +1,149 @@
+{
+ "test" : {
+ "mappings" : {
+ "_default_" : {
+ "dynamic_templates" : [
+ {
+ "message_field" : {
+ "path_match" : "message",
+ "match_mapping_type" : "string",
+ "mapping" : {
+ "norms" : false,
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "string_fields" : {
+ "match" : "*",
+ "match_mapping_type" : "string",
+ "mapping" : {
+ "fields" : {
+ "keyword" : {
+ "ignore_above" : 256,
+ "type" : "keyword"
+ }
+ },
+ "norms" : false,
+ "type" : "text"
+ }
+ }
+ }
+ ],
+ "properties" : {
+ "@timestamp" : {
+ "type" : "date"
+ },
+ "@version" : {
+ "type" : "keyword"
+ },
+ "geoip" : {
+ "dynamic" : "true",
+ "properties" : {
+ "ip" : {
+ "type" : "ip"
+ },
+ "latitude" : {
+ "type" : "half_float"
+ },
+ "location" : {
+ "type" : "geo_point"
+ },
+ "longitude" : {
+ "type" : "half_float"
+ }
+ }
+ }
+ }
+ },
+ "doc" : {
+ "dynamic_templates" : [
+ {
+ "message_field" : {
+ "path_match" : "message",
+ "match_mapping_type" : "string",
+ "mapping" : {
+ "norms" : false,
+ "type" : "text"
+ }
+ }
+ },
+ {
+ "string_fields" : {
+ "match" : "*",
+ "match_mapping_type" : "string",
+ "mapping" : {
+ "fields" : {
+ "keyword" : {
+ "ignore_above" : 256,
+ "type" : "keyword"
+ }
+ },
+ "norms" : false,
+ "type" : "text"
+ }
+ }
+ }
+ ],
+ "properties" : {
+ "@timestamp" : {
+ "type" : "date"
+ },
+ "@version" : {
+ "type" : "keyword"
+ },
+ "act" : {
+ "type" : "text",
+ "norms" : false,
+ "fields" : {
+ "keyword" : {
+ "type" : "keyword",
+ "ignore_above" : 256
+ }
+ }
+ },
+ "app" : {
+ "type" : "text",
+ "norms" : false,
+ "fields" : {
+ "keyword" : {
+ "type" : "keyword",
+ "ignore_above" : 256
+ }
+ }
+ },
+ "tags" : {
+ "type" : "text",
+ "norms" : false,
+ "fields" : {
+ "keyword" : {
+ "type" : "keyword",
+ "ignore_above" : 256
+ }
+ }
+ },
+ "type" : {
+ "type" : "text",
+ "norms" : false,
+ "fields" : {
+ "keyword" : {
+ "type" : "keyword",
+ "ignore_above" : 256
+ }
+ }
+ },
+ "uid" : {
+ "type" : "text",
+ "norms" : false,
+ "fields" : {
+ "keyword" : {
+ "type" : "keyword",
+ "ignore_above" : 256
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
