This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8b40791718 [Feature](ES): catalog support mapping es _id #15943
8b40791718 is described below
commit 8b40791718011c6acd5c5e885a2e0c75492c10f2
Author: Stalary <[email protected]>
AuthorDate: Sat Jan 21 08:08:32 2023 +0800
[Feature](ES): catalog support mapping es _id #15943
---
.../src/main/java/org/apache/doris/catalog/EsResource.java | 5 +++++
.../src/main/java/org/apache/doris/catalog/EsTable.java | 2 +-
.../org/apache/doris/datasource/EsExternalCatalog.java | 13 ++++++-------
.../org/apache/doris/external/elasticsearch/EsUtil.java | 14 +++++++++++++-
4 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
index 1b9d5863d0..012ad119e3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
@@ -59,12 +59,14 @@ public class EsResource extends Resource {
public static final String MAX_DOCVALUE_FIELDS = "max_docvalue_fields";
public static final String NODES_DISCOVERY = "nodes_discovery";
public static final String HTTP_SSL_ENABLED = "http_ssl_enabled";
+ public static final String MAPPING_ES_ID = "mapping_es_id";
public static final String QUERY_DSL = "query_dsl";
public static final String DOC_VALUE_SCAN_DEFAULT_VALUE = "true";
public static final String KEYWORD_SNIFF_DEFAULT_VALUE = "true";
public static final String HTTP_SSL_ENABLED_DEFAULT_VALUE = "false";
public static final String NODES_DISCOVERY_DEFAULT_VALUE = "true";
+ public static final String MAPPING_ES_ID_DEFAULT_VALUE = "false";
@SerializedName(value = "properties")
private Map<String, String> properties;
@@ -122,6 +124,9 @@ public class EsResource extends Resource {
if (properties.containsKey(EsResource.NODES_DISCOVERY)) {
EsUtil.getBoolean(properties, EsResource.NODES_DISCOVERY);
}
+ if (properties.containsKey(EsResource.MAPPING_ES_ID)) {
+ EsUtil.getBoolean(properties, EsResource.MAPPING_ES_ID);
+ }
}
private Map<String, String> processCompatibleProperties(Map<String,
String> props) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
index 0c89a3b0d6..a26a7db7f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java
@@ -306,6 +306,6 @@ public class EsTable extends Table {
}
public List<Column> genColumnsFromEs() {
- return EsUtil.genColumnsFromEs(client, indexName, mappingType);
+ return EsUtil.genColumnsFromEs(client, indexName, mappingType, false);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
index f3441e0c9f..9b219ea6a2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
@@ -67,12 +67,6 @@ public class EsExternalCatalog extends ExternalCatalog {
if (properties.containsKey("username")) {
properties.put(EsResource.USER, properties.remove("username"));
}
- if (properties.containsKey("doc_value_scan")) {
- properties.put(EsResource.DOC_VALUE_SCAN,
properties.remove("doc_value_scan"));
- }
- if (properties.containsKey("keyword_sniff")) {
- properties.put(EsResource.KEYWORD_SNIFF,
properties.remove("keyword_sniff"));
- }
return properties;
}
@@ -109,6 +103,11 @@ public class EsExternalCatalog extends ExternalCatalog {
EsResource.NODES_DISCOVERY_DEFAULT_VALUE));
}
+ public boolean enableMappingEsId() {
+ return
Boolean.parseBoolean(catalogProperty.getOrDefault(EsResource.MAPPING_ES_ID,
+ EsResource.MAPPING_ES_ID_DEFAULT_VALUE));
+ }
+
@Override
protected void initLocalObjectsImpl() {
esRestClient = new EsRestClient(getNodes(), getUsername(),
getPassword(), enableSsl());
@@ -161,6 +160,6 @@ public class EsExternalCatalog extends ExternalCatalog {
@Override
public List<Column> getSchema(String dbName, String tblName) {
makeSureInitialized();
- return EsUtil.genColumnsFromEs(getEsRestClient(), tblName, null);
+ return EsUtil.genColumnsFromEs(getEsRestClient(), tblName, null,
enableMappingEsId());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
index 0cde72067d..10c9a675c2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
@@ -189,6 +189,7 @@ public class EsUtil {
/**
* Remove `dynamic_templates` and check explicit mapping
+ *
* @param mappings
*/
private static void checkDynamicTemplates(JSONObject mappings) {
@@ -376,13 +377,24 @@ public class EsUtil {
/**
* Generate columns from ES Cluster.
+ * Add mappingEsId config in es external catalog.
**/
- public static List<Column> genColumnsFromEs(EsRestClient client, String
indexName, String mappingType) {
+ public static List<Column> genColumnsFromEs(EsRestClient client, String
indexName, String mappingType,
+ boolean mappingEsId) {
String mapping = client.getMapping(indexName);
JSONObject mappingProps = getMappingProps(indexName, mapping,
mappingType);
List<String> arrayFields = getArrayFields(mapping);
Set<String> keys = (Set<String>) mappingProps.keySet();
List<Column> columns = new ArrayList<>();
+ if (mappingEsId) {
+ Column column = new Column();
+ column.setName("_id");
+ column.setIsKey(true);
+ column.setType(ScalarType.createVarcharType(255));
+ column.setIsAllowNull(true);
+ column.setUniqueId(-1);
+ columns.add(column);
+ }
for (String key : keys) {
JSONObject field = (JSONObject) mappingProps.get(key);
Type type;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
