This is an automated email from the ASF dual-hosted git repository. shuber pushed a commit to branch UNOMI-225-ES7 in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 98d9f064ccbe97165fcba9d1d4956e043bd0e806 Author: Serge Huber <[email protected]> AuthorDate: Mon Nov 18 23:34:39 2019 +0100 UNOMI-225 ElasticSearch 7 support - Connection to ElasticSearch is now working, still lots of things to fix, notably in mappings. Signed-off-by: Serge Huber <[email protected]> --- .../ElasticSearchPersistenceServiceImpl.java | 47 +++++----------------- .../resources/META-INF/cxs/mappings/_default_.json | 22 ---------- .../resources/META-INF/cxs/mappings/campaign.json | 20 ++++++++- .../META-INF/cxs/mappings/campaignevent.json | 22 ++++++++-- .../resources/META-INF/cxs/mappings/event.json | 20 ++++++++- .../main/resources/META-INF/cxs/mappings/goal.json | 22 ++++++++-- .../META-INF/cxs/mappings/personaSession.json | 22 ++++++++-- .../resources/META-INF/cxs/mappings/profile.json | 22 ++++++++-- .../META-INF/cxs/mappings/propertyType.json | 22 ++++++++-- .../main/resources/META-INF/cxs/mappings/rule.json | 22 ++++++++-- .../resources/META-INF/cxs/mappings/scoring.json | 20 ++++++++- .../resources/META-INF/cxs/mappings/segment.json | 20 ++++++++- .../resources/META-INF/cxs/mappings/session.json | 22 ++++++++-- 13 files changed, 215 insertions(+), 88 deletions(-) diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java index d85ede2..f1949ed 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java @@ -349,21 +349,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, } } if (!indexExists) { - logger.info("{} index doesn't exist yet, creating it...", indexName); - Map<String, Object> indexMappings = new HashMap<>(); - indexMappings.put("_default_", mappings.get("_default_")); - for (Map.Entry<String, String> entry : mappings.entrySet()) { - if (!itemsMonthlyIndexed.contains(entry.getKey()) && !indexNames.containsKey(entry.getKey())) { - indexMappings.put(entry.getKey(), entry.getValue()); - } - } - - internalCreateIndex(indexName, indexMappings); + logger.info("{} index doesn't exist yet, creating it...", getIndex("profile", null)); + internalCreateIndex(getIndex("profile", null), mappings.get("profile")); } else { - logger.info("Found index {}, ElasticSearch started successfully.", indexName); - for (Map.Entry<String, String> entry : mappings.entrySet()) { - createMapping(entry.getKey(), entry.getValue()); - } + logger.info("Found index {}, ElasticSearch started successfully.", getIndex("profile", null)); + createMapping(getIndex("profile", null), mappings.get("profile")); } createMonthlyIndexTemplate(); @@ -915,8 +905,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, public boolean createMonthlyIndexTemplate() { Boolean result = new InClassLoaderExecute<Boolean>(metricsService, this.getClass().getName() + ".createMonthlyIndexTemplate") { protected Boolean execute(Object... args) throws IOException { - PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest("context-monthly-indices") - .patterns(Arrays.asList(indexName + "*" + INDEX_DATE_PREFIX + "*")) + PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest("context-event-monthly-indices") + .patterns(Arrays.asList(indexName + "-event-" + INDEX_DATE_PREFIX + "*")) .settings("{\n" + " \"index\" : {\n" + " \"number_of_shards\" : " + monthlyIndexNumberOfShards + ",\n" + @@ -932,14 +922,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, " }\n" + " }\n" + "}\n", XContentType.JSON); - Map<String, Object> indexMappings = new HashMap<String, Object>(); - indexMappings.put("_default_", mappings.get("_default_")); - for (Map.Entry<String, String> entry : mappings.entrySet()) { - if (itemsMonthlyIndexed.contains(entry.getKey())) { - indexMappings.put(entry.getKey(), entry.getValue()); - } - } - putIndexTemplateRequest.mapping(indexMappings); + putIndexTemplateRequest.mapping(mappings.get("event"), XContentType.JSON); AcknowledgedResponse putIndexTemplateResponse = client.indices().putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT); return putIndexTemplateResponse.isAcknowledged(); } @@ -957,14 +940,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, GetIndexRequest getIndexRequest = new GetIndexRequest(indexName); boolean indexExists = client.indices().exists(getIndexRequest, RequestOptions.DEFAULT); if (!indexExists) { - Map<String, Object> indexMappings = new HashMap<String, Object>(); - indexMappings.put("_default_", mappings.get("_default_")); - for (Map.Entry<String, String> entry : mappings.entrySet()) { - if (indexNames.containsKey(entry.getKey()) && indexNames.get(entry.getKey()).equals(indexName)) { - indexMappings.put(entry.getKey(), entry.getValue()); - } - } - internalCreateIndex(indexName, indexMappings); + internalCreateIndex(indexName, mappings.get(indexName)); } return !indexExists; } @@ -995,7 +971,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, } } - private void internalCreateIndex(String indexName, Map<String, Object> mappings) throws IOException { + private void internalCreateIndex(String indexName, String mappingSource) throws IOException { CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); createIndexRequest.settings("{\n" + " \"index\" : {\n" + @@ -1013,7 +989,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, " }\n" + "}\n", XContentType.JSON); - createIndexRequest.mapping(mappings); + createIndexRequest.mapping(mappingSource, XContentType.JSON); client.indices().create(createIndexRequest, RequestOptions.DEFAULT); } @@ -1026,9 +1002,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, @Override public void createMapping(String type, String source) { - if (type.equals("_default_")) { - return; - } try { if (itemsMonthlyIndexed.contains(type)) { createMonthlyIndexTemplate(); diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json deleted file mode 100644 index 9bc4ab0..0000000 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "_default_": { - "dynamic_templates": [ - { - "all": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "text", - "analyzer": "folding", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - ] - } -} diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json index 3e36a7f..6290fb6 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json @@ -1,5 +1,22 @@ { - "campaign": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], "properties": { "cost": { "type": "double" @@ -27,5 +44,4 @@ } } } - } } \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json index 9ab8779..9b05a7f 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json @@ -1,6 +1,23 @@ { - "campaignevent": { - "properties": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { "cost": { "type": "double" }, @@ -24,5 +41,4 @@ } } } - } } \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json index 69c9f08..829091a 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json @@ -1,10 +1,26 @@ { - "event": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], "properties": { "timeStamp": { "type": "date" } } - } } diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json index 3e0bb6f..9627340 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json @@ -1,6 +1,23 @@ { - "goal": { - "properties": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { "metadata": { "properties": { @@ -19,5 +36,4 @@ } } } - } } \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json index ce3a8f3..901266d 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json @@ -1,6 +1,23 @@ { - "personaSession": { - "properties": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { "duration" : { "type" : "long" }, @@ -22,5 +39,4 @@ "type" : "long" } } - } } \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json index 29d40b7..2fef62f 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json @@ -1,6 +1,23 @@ { - "profile": { - "properties": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { "properties": { "properties": { @@ -19,5 +36,4 @@ } } } - } } \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json index 99969f7..6adbd2a 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json @@ -1,6 +1,23 @@ { - "propertyType": { - "properties": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { "dateRanges": { "properties": { } @@ -41,5 +58,4 @@ "type": "double" } } - } } \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json index ea6bb26..3561e61 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json @@ -1,6 +1,23 @@ { - "rule": { - "properties": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { "metadata": { "properties": { "enabled": { @@ -27,5 +44,4 @@ "type": "boolean" } } - } } \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json index cda229b..8e256dd 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json @@ -1,5 +1,22 @@ { - "scoring": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], "properties": { "metadata": { @@ -19,5 +36,4 @@ } } } - } } diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json index 5e20436..fa93d5b 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json @@ -1,5 +1,22 @@ { - "segment": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], "properties": { "metadata": { @@ -20,5 +37,4 @@ } } } - } } diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json index d687dce..7cc4061 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json @@ -1,6 +1,23 @@ { - "session": { - "properties": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { "duration": { "type": "long" }, @@ -24,5 +41,4 @@ "type": "date" } } - } } \ No newline at end of file
