This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2b918eaccd [fix](Doris On ES) Fix es not support aliases error (#11547)
2b918eaccd is described below
commit 2b918eaccd8e716bccb6dae4cf16ba57a3982f43
Author: Stalary <[email protected]>
AuthorDate: Tue Aug 9 09:36:05 2022 +0800
[fix](Doris On ES) Fix es not support aliases error (#11547)
1. Fix es not support aliases error
2. Fix multicatalog query es error
3. add ut
---
.../java/org/apache/doris/catalog/EsTable.java | 9 +-
.../org/apache/doris/common/util/JsonUtil.java | 10 +-
.../doris/datasource/EsExternalDataSource.java | 8 +-
.../doris/external/elasticsearch/EsRestClient.java | 38 ++++-
.../doris/external/elasticsearch/EsUtil.java | 160 ++++-----------------
.../doris/external/elasticsearch/MappingPhase.java | 88 +++++++++++-
.../java/org/apache/doris/planner/EsScanNode.java | 20 +--
.../doris/external/elasticsearch/EsUtilTest.java | 102 +++++++------
.../resources/data/es/es6_aliases_mapping.json | 54 +++++++
.../test/resources/data/es/es6_index_mapping.json | 28 ++++
.../resources/data/es/es7_aliases_mapping.json | 50 +++++++
.../test/resources/data/es/es7_index_mapping.json | 26 ++++
.../src/test/resources/data/es/es7_mappings.json | 23 ---
.../resources/data/es/es8_aliases_mapping.json | 50 +++++++
.../test/resources/data/es/es8_index_mapping.json | 26 ++++
15 files changed, 470 insertions(+), 222 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
index 93dd2bc2bf..e7a1123aa1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
@@ -67,10 +67,6 @@ public class EsTable extends Table {
public static final String NODES_DISCOVERY = "nodes_discovery";
public static final String HTTP_SSL_ENABLED = "http_ssl_enabled";
public static final String ES_DSL = "es_dsl";
- public static final String INIT_SCROLL_URL = "init_scroll_url";
- public static final String NEXT_SCROLL_URL = "next_scroll_url";
- public static final String SEARCH_URL = "search_url";
-
private static final Logger LOG = LogManager.getLogger(EsTable.class);
// Solr doc_values vs stored_fields performance-smackdown indicate:
@@ -345,8 +341,9 @@ public class EsTable extends Table {
esMetaStateTracker.run();
this.esTablePartitions =
esMetaStateTracker.searchContext().tablePartitions();
} catch (Throwable e) {
- LOG.warn("Exception happens when fetch index [{}] meta data from
remote es cluster."
- + "table id: {}, err: ", this.name, this.id, e);
+ LOG.warn(
+ "Exception happens when fetch index [{}] meta data from
remote es cluster." + "table id: {}, err: ",
+ this.name, this.id, e);
this.esTablePartitions = null;
this.lastMetaDataSyncException = e;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/JsonUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/JsonUtil.java
index 76d913260f..1cd52b5067 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/JsonUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/JsonUtil.java
@@ -47,11 +47,19 @@ public class JsonUtil {
return objectMapper.valueToTree(obj);
}
+ public static JsonNode readTree(String str) {
+ try {
+ return objectMapper.readTree(str);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException("readTree exception.", e);
+ }
+ }
+
public static ArrayNode parseArray(String text) {
try {
return (ArrayNode) objectMapper.readTree(text);
} catch (Exception e) {
- throw new RuntimeException("Json deserialization exception.", e);
+ throw new RuntimeException("parseArray exception.", e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
index d531b4f153..bfa4b56010 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
@@ -44,7 +44,7 @@ import java.util.Map;
@Getter
public class EsExternalDataSource extends ExternalDataSource {
- public static final String DEFAULT_DB = "default";
+ public static final String DEFAULT_DB = "default_db";
private static final Logger LOG =
LogManager.getLogger(EsExternalDataSource.class);
private static final String PROP_HOSTS = "elasticsearch.hosts";
private static final String PROP_USERNAME = "elasticsearch.username";
@@ -64,9 +64,9 @@ public class EsExternalDataSource extends ExternalDataSource {
private String[] nodes;
- private String username = "";
+ private String username = null;
- private String password = "";
+ private String password = null;
private boolean enableDocValueScan = true;
@@ -166,7 +166,7 @@ public class EsExternalDataSource extends
ExternalDataSource {
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
- return esRestClient.getIndexes();
+ return esRestClient.listTable();
}
@Nullable
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
index 77fbe3bff6..eb79f4453b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
@@ -19,7 +19,9 @@ package org.apache.doris.external.elasticsearch;
import org.apache.doris.common.util.JsonUtil;
+import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.google.common.collect.ImmutableList;
import okhttp3.Credentials;
import okhttp3.OkHttpClient;
import okhttp3.Request;
@@ -36,9 +38,11 @@ import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
@@ -141,7 +145,7 @@ public class EsRestClient {
/**
* Get all index.
**/
- public List<String> getIndexes() {
+ public List<String> getIndices() {
String indexes =
execute("_cat/indices?h=index&format=json&s=index:asc");
if (indexes == null) {
throw new DorisEsException("get es indexes error");
@@ -158,6 +162,38 @@ public class EsRestClient {
return ret;
}
+ /**
+ * Get all alias.
+ **/
+ public Map<String, List<String>> getAliases() {
+ String res = execute("_aliases");
+ Map<String, List<String>> ret = new HashMap<>();
+ JsonNode root = JsonUtil.readTree(res);
+ if (root == null) {
+ return ret;
+ }
+ Iterator<Map.Entry<String, JsonNode>> elements = root.fields();
+ while (elements.hasNext()) {
+ Map.Entry<String, JsonNode> element = elements.next();
+ JsonNode aliases = element.getValue().get("aliases");
+ Iterator<String> aliasNames = aliases.fieldNames();
+ if (aliasNames.hasNext()) {
+ ret.put(element.getKey(), ImmutableList.copyOf(aliasNames));
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Returns the merge of index and alias
+ **/
+ public List<String> listTable() {
+ List<String> indices =
getIndices().stream().distinct().collect(Collectors.toList());
+ getAliases().entrySet().stream().filter(e ->
indices.contains(e.getKey()))
+ .flatMap(e ->
e.getValue().stream()).distinct().forEach(indices::add);
+ return indices;
+ }
+
/**
* Get Shard location.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
index a99a2cf598..4385977216 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
@@ -38,14 +38,12 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.EsTable;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.external.elasticsearch.QueryBuilders.QueryBuilder;
import org.apache.doris.thrift.TExprOpcode;
-import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.simple.JSONObject;
@@ -133,6 +131,9 @@ public class EsUtil {
}
}
+ /**
+ * Get Array fields.
+ **/
public static List<String> getArrayFields(String indexMapping) {
JSONObject mappings = getMapping(indexMapping);
if (!mappings.containsKey("_meta")) {
@@ -148,40 +149,42 @@ public class EsUtil {
private static JSONObject getMapping(String indexMapping) {
JSONObject jsonObject = (JSONObject) JSONValue.parse(indexMapping);
- // the indexName use alias takes the first mapping
+ // If the indexName use alias takes the first mapping
Iterator<String> keys = jsonObject.keySet().iterator();
String docKey = keys.next();
JSONObject docData = (JSONObject) jsonObject.get(docKey);
return (JSONObject) docData.get("mappings");
}
+ private static JSONObject getRootSchema(JSONObject mappings, String
mappingType) {
+ // Type is null in the following three cases
+ // 1. Equal 6.8.x and after
+ // 2. Multi-catalog auto infer
+ // 3. Equal 6.8.x and before user not passed
+ if (mappingType == null) {
+ String firstType = (String) mappings.keySet().iterator().next();
+ if (!"properties".equals(firstType)) {
+ // If type is not passed in takes the first type.
+ return (JSONObject) mappings.get(firstType);
+ }
+ // Equal 7.x and after
+ return mappings;
+ } else {
+ if (mappings.containsKey(mappingType)) {
+ return (JSONObject) mappings.get(mappingType);
+ }
+ // Compatible type error
+ return getRootSchema(mappings, null);
+ }
+ }
+
/**
* Get mapping properties JSONObject.
**/
public static JSONObject getMappingProps(String sourceIndex, String
indexMapping, String mappingType) {
JSONObject mappings = getMapping(indexMapping);
- JSONObject rootSchema = (JSONObject) mappings.get(mappingType);
- JSONObject properties;
- // Elasticsearch 7.x, type was removed from ES mapping, default type
is `_doc`
- //
https://www.elastic.co/guide/en/elasticsearch/reference/7.0/removal-of-types.html
- // Elasticsearch 8.x, include_type_name parameter is removed
- if (rootSchema == null) {
- properties = (JSONObject) mappings.get("properties");
- // Compatible es6 with no type passed in.
- if (mappingType == null) {
- String typeKey = (String) mappings.keySet().iterator().next();
- JSONObject typeProps = (JSONObject) ((JSONObject)
mappings.get(typeKey)).get("properties");
- if (typeProps != null) {
- properties = typeProps;
- if (properties.containsKey("mappings")) {
- properties.remove("mappings");
- properties.remove("settings");
- }
- }
- }
- } else {
- properties = (JSONObject) rootSchema.get("properties");
- }
+ JSONObject rootSchema = getRootSchema(mappings, mappingType);
+ JSONObject properties = (JSONObject) rootSchema.get("properties");
if (properties == null) {
throw new DorisEsException(
"index[" + sourceIndex + "] type[" + mappingType + "]
mapping not found for the ES Cluster");
@@ -189,85 +192,6 @@ public class EsUtil {
return properties;
}
- /**
- * Parse the required field information from the json.
- *
- * @param searchContext the current associated column searchContext
- * @param indexMapping the return value of _mapping
- */
- public static void resolveFields(SearchContext searchContext, String
indexMapping) throws DorisEsException {
- JSONObject properties = getMappingProps(searchContext.sourceIndex(),
indexMapping, searchContext.type());
- for (Column col : searchContext.columns()) {
- String colName = col.getName();
- // if column exists in Doris Table but no found in ES's mapping,
we choose to ignore this situation?
- if (!properties.containsKey(colName)) {
- throw new DorisEsException(
- "index[" + searchContext.sourceIndex() + "] type[" +
indexMapping + "] mapping not found column"
- + colName + " for the ES Cluster");
- }
- JSONObject fieldObject = (JSONObject) properties.get(colName);
- resolveKeywordFields(searchContext, fieldObject, colName);
- resolveDocValuesFields(searchContext, fieldObject, colName);
- }
- }
-
- // get a field of keyword type in the fields
- private static void resolveKeywordFields(SearchContext searchContext,
JSONObject fieldObject, String colName) {
- String fieldType = (String) fieldObject.get("type");
- // string-type field used keyword type to generate predicate
- // if text field type seen, we should use the `field` keyword type?
- if ("text".equals(fieldType)) {
- JSONObject fieldsObject = (JSONObject) fieldObject.get("fields");
- if (fieldsObject != null) {
- for (Object key : fieldsObject.keySet()) {
- JSONObject innerTypeObject = (JSONObject)
fieldsObject.get((String) key);
- // just for text type
- if ("keyword".equals((String)
innerTypeObject.get("type"))) {
- searchContext.fetchFieldsContext().put(colName,
colName + "." + key);
- }
- }
- }
- }
- }
-
- private static void resolveDocValuesFields(SearchContext searchContext,
JSONObject fieldObject, String colName) {
- String fieldType = (String) fieldObject.get("type");
- String docValueField = null;
- if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) {
- JSONObject fieldsObject = (JSONObject) fieldObject.get("fields");
- if (fieldsObject != null) {
- for (Object key : fieldsObject.keySet()) {
- JSONObject innerTypeObject = (JSONObject)
fieldsObject.get((String) key);
- if
(EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains((String)
innerTypeObject.get("type"))) {
- continue;
- }
- if (innerTypeObject.containsKey("doc_values")) {
- boolean docValue = (Boolean)
innerTypeObject.get("doc_values");
- if (docValue) {
- docValueField = colName;
- }
- } else {
- // a : {c : {}} -> a -> a.c
- docValueField = colName + "." + key;
- }
- }
- }
- } else {
- // set doc_value = false manually
- if (fieldObject.containsKey("doc_values")) {
- Boolean docValue = (Boolean) fieldObject.get("doc_values");
- if (!docValue) {
- return;
- }
- }
- docValueField = colName;
- }
- // docValueField Cannot be null
- if (StringUtils.isNotEmpty(docValueField)) {
- searchContext.docValueFieldsContext().put(colName, docValueField);
- }
- }
-
private static QueryBuilder toCompoundEsDsl(Expr expr) {
CompoundPredicate compoundPredicate = (CompoundPredicate) expr;
switch (compoundPredicate.getOp()) {
@@ -464,7 +388,7 @@ public class EsUtil {
return boolLiteral.getValue();
} else if (expr instanceof DateLiteral) {
DateLiteral dateLiteral = (DateLiteral) expr;
- return dateLiteral.getLongValue();
+ return dateLiteral.getStringValue();
} else if (expr instanceof DecimalLiteral) {
DecimalLiteral decimalLiteral = (DecimalLiteral) expr;
return decimalLiteral.getValue();
@@ -484,30 +408,4 @@ public class EsUtil {
return null;
}
- /**
- * Generate url for be to query es.
- **/
- public static EsUrls genEsUrls(String index, String type, boolean
docValueMode, long limit, long batchSize) {
- String filterPath = docValueMode ?
"filter_path=_scroll_id,hits.total,hits.hits._score,hits.hits.fields"
- :
"filter_path=_scroll_id,hits.hits._source,hits.total,hits.hits._id";
- if (limit <= 0) {
- StringBuilder initScrollUrl = new StringBuilder();
- StringBuilder nextScrollUrl = new StringBuilder();
- initScrollUrl.append("/").append(index);
- if (StringUtils.isNotBlank(type)) {
- initScrollUrl.append("/").append(type);
- }
-
initScrollUrl.append("/_search?").append(filterPath).append("&terminate_after=").append(batchSize);
- nextScrollUrl.append("/_search/scroll?").append(filterPath);
- return new EsUrls(null, initScrollUrl.toString(),
nextScrollUrl.toString());
- } else {
- StringBuilder searchUrl = new StringBuilder();
- searchUrl.append("/").append(index);
- if (StringUtils.isNotBlank(type)) {
- searchUrl.append("/").append(type);
- }
-
searchUrl.append("/_search?terminate_after=").append(limit).append("&").append(filterPath);
- return new EsUrls(searchUrl.toString(), null, null);
- }
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
index 560cebf1ff..d0082c45e8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java
@@ -17,6 +17,12 @@
package org.apache.doris.external.elasticsearch;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.EsTable;
+
+import org.apache.commons.lang3.StringUtils;
+import org.json.simple.JSONObject;
+
/**
* Get index mapping from remote ES Cluster, and resolved `keyword` and
`doc_values` field
* Later we can use it to parse all relevant indexes
@@ -39,7 +45,87 @@ public class MappingPhase implements SearchPhase {
@Override
public void postProcess(SearchContext context) {
- EsUtil.resolveFields(context, jsonMapping);
+ resolveFields(context, jsonMapping);
+ }
+
+ /**
+ * Parse the required field information from the json.
+ *
+ * @param searchContext the current associated column searchContext
+ * @param indexMapping the return value of _mapping
+ */
+ public static void resolveFields(SearchContext searchContext, String
indexMapping) throws DorisEsException {
+ JSONObject properties =
EsUtil.getMappingProps(searchContext.sourceIndex(), indexMapping,
searchContext.type());
+ for (Column col : searchContext.columns()) {
+ String colName = col.getName();
+ // if column exists in Doris Table but no found in ES's mapping,
we choose to ignore this situation?
+ if (!properties.containsKey(colName)) {
+ throw new DorisEsException(
+ "index[" + searchContext.sourceIndex() + "] type[" +
indexMapping + "] mapping not found column"
+ + colName + " for the ES Cluster");
+ }
+ JSONObject fieldObject = (JSONObject) properties.get(colName);
+ resolveKeywordFields(searchContext, fieldObject, colName);
+ resolveDocValuesFields(searchContext, fieldObject, colName);
+ }
+ }
+
+
+ // get a field of keyword type in the fields
+ private static void resolveKeywordFields(SearchContext searchContext,
JSONObject fieldObject, String colName) {
+ String fieldType = (String) fieldObject.get("type");
+ // string-type field used keyword type to generate predicate
+ // if text field type seen, we should use the `field` keyword type?
+ if ("text".equals(fieldType)) {
+ JSONObject fieldsObject = (JSONObject) fieldObject.get("fields");
+ if (fieldsObject != null) {
+ for (Object key : fieldsObject.keySet()) {
+ JSONObject innerTypeObject = (JSONObject)
fieldsObject.get((String) key);
+ // just for text type
+ if ("keyword".equals((String)
innerTypeObject.get("type"))) {
+ searchContext.fetchFieldsContext().put(colName,
colName + "." + key);
+ }
+ }
+ }
+ }
+ }
+
+ private static void resolveDocValuesFields(SearchContext searchContext,
JSONObject fieldObject, String colName) {
+ String fieldType = (String) fieldObject.get("type");
+ String docValueField = null;
+ if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) {
+ JSONObject fieldsObject = (JSONObject) fieldObject.get("fields");
+ if (fieldsObject != null) {
+ for (Object key : fieldsObject.keySet()) {
+ JSONObject innerTypeObject = (JSONObject)
fieldsObject.get((String) key);
+ if
(EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains((String)
innerTypeObject.get("type"))) {
+ continue;
+ }
+ if (innerTypeObject.containsKey("doc_values")) {
+ boolean docValue = (Boolean)
innerTypeObject.get("doc_values");
+ if (docValue) {
+ docValueField = colName;
+ }
+ } else {
+ // a : {c : {}} -> a -> a.c
+ docValueField = colName + "." + key;
+ }
+ }
+ }
+ } else {
+ // set doc_value = false manually
+ if (fieldObject.containsKey("doc_values")) {
+ Boolean docValue = (Boolean) fieldObject.get("doc_values");
+ if (!docValue) {
+ return;
+ }
+ }
+ docValueField = colName;
+ }
+ // docValueField Cannot be null
+ if (StringUtils.isNotEmpty(docValueField)) {
+ searchContext.docValueFieldsContext().put(colName, docValueField);
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
index d2ff066c22..11f3e1cb2b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
@@ -32,12 +32,10 @@ import org.apache.doris.common.UserException;
import org.apache.doris.external.elasticsearch.EsShardPartitions;
import org.apache.doris.external.elasticsearch.EsShardRouting;
import org.apache.doris.external.elasticsearch.EsTablePartitions;
-import org.apache.doris.external.elasticsearch.EsUrls;
import org.apache.doris.external.elasticsearch.EsUtil;
import org.apache.doris.external.elasticsearch.QueryBuilders;
import org.apache.doris.external.elasticsearch.QueryBuilders.BoolQueryBuilder;
import org.apache.doris.external.elasticsearch.QueryBuilders.QueryBuilder;
-import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TEsScanNode;
@@ -167,8 +165,12 @@ public class EsScanNode extends ScanNode {
buildQuery();
msg.node_type = TPlanNodeType.ES_HTTP_SCAN_NODE;
Map<String, String> properties = Maps.newHashMap();
- properties.put(EsTable.USER, table.getUserName());
- properties.put(EsTable.PASSWORD, table.getPasswd());
+ if (table.getUserName() != null) {
+ properties.put(EsTable.USER, table.getUserName());
+ }
+ if (table.getPasswd() != null) {
+ properties.put(EsTable.PASSWORD, table.getPasswd());
+ }
properties.put(EsTable.HTTP_SSL_ENABLED,
String.valueOf(table.isHttpSslEnabled()));
TEsScanNode esScanNode = new TEsScanNode(desc.getId().asInt());
esScanNode.setProperties(properties);
@@ -177,16 +179,6 @@ public class EsScanNode extends ScanNode {
properties.put(EsTable.DOC_VALUES_MODE,
String.valueOf(useDocValueScan(desc, table.docValueContext())));
}
properties.put(EsTable.ES_DSL, queryBuilder.toJson());
-
- // Be use it add es host_port and shardId to query.
- EsUrls esUrls = EsUtil.genEsUrls(table.getIndexName(),
table.getMappingType(), table.isEnableDocValueScan(),
- ConnectContext.get().getSessionVariable().batchSize,
msg.limit);
- if (esUrls.getSearchUrl() != null) {
- properties.put(EsTable.SEARCH_URL, esUrls.getSearchUrl());
- } else {
- properties.put(EsTable.INIT_SCROLL_URL, esUrls.getInitScrollUrl());
- properties.put(EsTable.NEXT_SCROLL_URL, esUrls.getNextScrollUrl());
- }
if (table.isEnableKeywordSniff() && table.fieldsContext().size() > 0) {
esScanNode.setFieldsContext(table.fieldsContext());
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
index 93fc870db4..432af95c69 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
@@ -42,6 +42,8 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;
+import java.io.IOException;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
@@ -81,7 +83,7 @@ public class EsUtilTest extends EsTestCase {
// ES version < 7.0
EsTable esTableBefore7X = fakeEsTable("fake", "test", "doc", columns);
SearchContext searchContext = new SearchContext(esTableBefore7X);
- EsUtil.resolveFields(searchContext,
loadJsonFromFile("data/es/test_index_mapping.json"));
+ MappingPhase.resolveFields(searchContext,
loadJsonFromFile("data/es/test_index_mapping.json"));
Assert.assertEquals("k3.keyword",
searchContext.fetchFieldsContext().get("k3"));
Assert.assertEquals("k3.keyword",
searchContext.docValueFieldsContext().get("k3"));
Assert.assertEquals("k1",
searchContext.docValueFieldsContext().get("k1"));
@@ -90,22 +92,13 @@ public class EsUtilTest extends EsTestCase {
// ES version >= 7.0
EsTable esTableAfter7X = fakeEsTable("fake", "test", "_doc", columns);
SearchContext searchContext1 = new SearchContext(esTableAfter7X);
- EsUtil.resolveFields(searchContext1,
loadJsonFromFile("data/es/test_index_mapping_after_7x.json"));
+ MappingPhase.resolveFields(searchContext1,
loadJsonFromFile("data/es/test_index_mapping_after_7x.json"));
Assert.assertEquals("k3.keyword",
searchContext1.fetchFieldsContext().get("k3"));
Assert.assertEquals("k3.keyword",
searchContext1.docValueFieldsContext().get("k3"));
Assert.assertEquals("k1",
searchContext1.docValueFieldsContext().get("k1"));
Assert.assertEquals("k2",
searchContext1.docValueFieldsContext().get("k2"));
}
- @Test
- public void testTypeNotExist() throws Exception {
- EsTable table = fakeEsTable("fake", "test", "not_exists", columns);
- SearchContext searchContext = new SearchContext(table);
- // type not exists
- ExceptionChecker.expectThrows(DorisEsException.class,
- () -> EsUtil.resolveFields(searchContext,
loadJsonFromFile("data/es/test_index_mapping.json")));
- }
-
@Test
public void testWorkFlow(@Injectable EsRestClient client) throws Exception
{
EsTable table = fakeEsTable("fake", "test", "doc", columns);
@@ -132,7 +125,8 @@ public class EsUtilTest extends EsTestCase {
public void testMultTextFields() throws Exception {
EsTable esTableAfter7X = fakeEsTable("fake", "test", "_doc", columns);
SearchContext searchContext = new SearchContext(esTableAfter7X);
- EsUtil.resolveFields(searchContext,
loadJsonFromFile("data/es/test_index_mapping_field_mult_analyzer.json"));
+ MappingPhase.resolveFields(searchContext,
+
loadJsonFromFile("data/es/test_index_mapping_field_mult_analyzer.json"));
Assert.assertFalse(searchContext.docValueFieldsContext().containsKey("k3"));
}
@@ -258,36 +252,62 @@ public class EsUtilTest extends EsTestCase {
}
@Test
- public void testGenEsUrls() {
- EsUrls typeLimit = EsUtil.genEsUrls("test", "_doc", false, 10, 1024);
- Assertions.assertEquals(
-
"/test/_doc/_search?terminate_after=10&filter_path=_scroll_id,hits.hits._source,hits.total,hits.hits._id",
- typeLimit.getSearchUrl());
- Assertions.assertNull(typeLimit.getInitScrollUrl());
- Assertions.assertNull(typeLimit.getNextScrollUrl());
-
- Assertions.assertEquals(
-
"/test/_search?terminate_after=10&filter_path=_scroll_id,hits.hits._source,hits.total,hits.hits._id",
- EsUtil.genEsUrls("test", null, false, 10,
1024).getSearchUrl());
-
- EsUrls typeNoLimit = EsUtil.genEsUrls("test", "_doc", false, -1, 1024);
- Assertions.assertEquals(
-
"/test/_doc/_search?filter_path=_scroll_id,hits.hits._source,hits.total,hits.hits._id&terminate_after=1024",
- typeNoLimit.getInitScrollUrl());
-
Assertions.assertEquals("/_search/scroll?filter_path=_scroll_id,hits.hits._source,hits.total,hits.hits._id",
- typeNoLimit.getNextScrollUrl());
- Assertions.assertNull(typeNoLimit.getSearchUrl());
+ public void testEs6Mapping() throws IOException, URISyntaxException {
+ JSONObject testAliases = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es6_aliases_mapping.json"),
+ "doc");
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testAliases.toJSONString());
+ JSONObject testAliasesNoType = EsUtil.getMappingProps("test",
+ loadJsonFromFile("data/es/es6_aliases_mapping.json"), null);
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
+ testAliasesNoType.toJSONString());
+ JSONObject testIndex = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es6_index_mapping.json"),
+ "doc");
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testIndex.toJSONString());
+ }
- EsUrls noTypeNoLimit = EsUtil.genEsUrls("test", null, false, -1, 2048);
- Assertions.assertEquals(
-
"/test/_search?filter_path=_scroll_id,hits.hits._source,hits.total,hits.hits._id&terminate_after=2048",
- noTypeNoLimit.getInitScrollUrl());
-
Assertions.assertEquals("/_search/scroll?filter_path=_scroll_id,hits.hits._source,hits.total,hits.hits._id",
- noTypeNoLimit.getNextScrollUrl());
+ @Test
+ public void testEs7Mapping() throws IOException, URISyntaxException {
+ JSONObject testAliases = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es7_aliases_mapping.json"),
+ null);
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testAliases.toJSONString());
+ JSONObject testAliasesErrorType = EsUtil.getMappingProps("test",
+ loadJsonFromFile("data/es/es7_aliases_mapping.json"), "doc");
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
+ testAliasesErrorType.toJSONString());
+ JSONObject testIndex = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es7_index_mapping.json"),
+ "doc");
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testIndex.toJSONString());
+ }
- EsUrls docValueTypeLimit = EsUtil.genEsUrls("test", "_doc", true, 100,
1024);
- Assertions.assertEquals(
-
"/test/_doc/_search?terminate_after=100&filter_path=_scroll_id,hits.total,hits.hits._score,hits.hits.fields",
- docValueTypeLimit.getSearchUrl());
+ @Test
+ public void testEs8Mapping() throws IOException, URISyntaxException {
+ JSONObject testAliases = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es8_aliases_mapping.json"),
+ null);
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testAliases.toJSONString());
+ JSONObject testAliasesErrorType = EsUtil.getMappingProps("test",
+ loadJsonFromFile("data/es/es8_aliases_mapping.json"), "doc");
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
+ testAliasesErrorType.toJSONString());
+ JSONObject testIndex = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es8_index_mapping.json"),
+ "doc");
+
Assertions.assertEquals("{\"test4\":{\"type\":\"date\"},\"test2\":{\"type\":\"text\","
+ +
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+ +
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testIndex.toJSONString());
}
}
diff --git a/fe/fe-core/src/test/resources/data/es/es6_aliases_mapping.json
b/fe/fe-core/src/test/resources/data/es/es6_aliases_mapping.json
new file mode 100644
index 0000000000..306e57f387
--- /dev/null
+++ b/fe/fe-core/src/test/resources/data/es/es6_aliases_mapping.json
@@ -0,0 +1,54 @@
+{
+ "test_202208": {
+ "mappings": {
+ "doc": {
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ }
+ },
+ "test_202207": {
+ "mappings": {
+ "doc": {
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/fe/fe-core/src/test/resources/data/es/es6_index_mapping.json
b/fe/fe-core/src/test/resources/data/es/es6_index_mapping.json
new file mode 100644
index 0000000000..8fa6a84907
--- /dev/null
+++ b/fe/fe-core/src/test/resources/data/es/es6_index_mapping.json
@@ -0,0 +1,28 @@
+{
+ "test_202207": {
+ "mappings": {
+ "doc": {
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/fe/fe-core/src/test/resources/data/es/es7_aliases_mapping.json
b/fe/fe-core/src/test/resources/data/es/es7_aliases_mapping.json
new file mode 100644
index 0000000000..4a0112e319
--- /dev/null
+++ b/fe/fe-core/src/test/resources/data/es/es7_aliases_mapping.json
@@ -0,0 +1,50 @@
+{
+ "test_202208": {
+ "mappings": {
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ },
+ "test_202207": {
+ "mappings": {
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/fe/fe-core/src/test/resources/data/es/es7_index_mapping.json
b/fe/fe-core/src/test/resources/data/es/es7_index_mapping.json
new file mode 100644
index 0000000000..93b3a22887
--- /dev/null
+++ b/fe/fe-core/src/test/resources/data/es/es7_index_mapping.json
@@ -0,0 +1,26 @@
+{
+ "test_202207": {
+ "mappings": {
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/fe/fe-core/src/test/resources/data/es/es7_mappings.json
b/fe/fe-core/src/test/resources/data/es/es7_mappings.json
deleted file mode 100644
index cf44130f68..0000000000
--- a/fe/fe-core/src/test/resources/data/es/es7_mappings.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "indexa_2020.05.02": {
- "mappings": {
- "dynamic": "strict",
- "properties": {
- "time": {
- "type": "long"
- },
- "type": {
- "type": "keyword"
- },
- "userId": {
- "type": "text",
- "fields": {
- "keyword": {
- "type": "keyword"
- }
- }
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/fe/fe-core/src/test/resources/data/es/es8_aliases_mapping.json
b/fe/fe-core/src/test/resources/data/es/es8_aliases_mapping.json
new file mode 100644
index 0000000000..3cfbdfe719
--- /dev/null
+++ b/fe/fe-core/src/test/resources/data/es/es8_aliases_mapping.json
@@ -0,0 +1,50 @@
+{
+ "test_202207": {
+ "mappings": {
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ },
+ "test_202208": {
+ "mappings": {
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/fe/fe-core/src/test/resources/data/es/es8_index_mapping.json
b/fe/fe-core/src/test/resources/data/es/es8_index_mapping.json
new file mode 100644
index 0000000000..93b3a22887
--- /dev/null
+++ b/fe/fe-core/src/test/resources/data/es/es8_index_mapping.json
@@ -0,0 +1,26 @@
+{
+ "test_202207": {
+ "mappings": {
+ "properties": {
+ "test1": {
+ "type": "keyword"
+ },
+ "test2": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ },
+ "test3": {
+ "type": "double"
+ },
+ "test4": {
+ "type": "date"
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
