This is an automated email from the ASF dual-hosted git repository.
shuber pushed a commit to branch UNOMI-225-ES7
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/UNOMI-225-ES7 by this push:
new 30e644b UNOMI-225 ElasticSearch 7 support - Connection to
ElasticSearch is now working, still lots of things to fix, notably in mappings.
30e644b is described below
commit 30e644b84c6505c3e7ba2c7244fba0f984ba080b
Author: Serge Huber <[email protected]>
AuthorDate: Mon Nov 18 23:34:39 2019 +0100
UNOMI-225 ElasticSearch 7 support
- Connection to ElasticSearch is now working, still lots of things to fix,
notably in mappings.
Signed-off-by: Serge Huber <[email protected]>
---
.../ElasticSearchPersistenceServiceImpl.java | 47 +++++-----------------
.../resources/META-INF/cxs/mappings/_default_.json | 22 ----------
.../resources/META-INF/cxs/mappings/campaign.json | 20 ++++++++-
.../META-INF/cxs/mappings/campaignevent.json | 22 ++++++++--
.../resources/META-INF/cxs/mappings/event.json | 20 ++++++++-
.../main/resources/META-INF/cxs/mappings/goal.json | 22 ++++++++--
.../META-INF/cxs/mappings/personaSession.json | 22 ++++++++--
.../resources/META-INF/cxs/mappings/profile.json | 22 ++++++++--
.../META-INF/cxs/mappings/propertyType.json | 22 ++++++++--
.../main/resources/META-INF/cxs/mappings/rule.json | 22 ++++++++--
.../resources/META-INF/cxs/mappings/scoring.json | 20 ++++++++-
.../resources/META-INF/cxs/mappings/segment.json | 20 ++++++++-
.../resources/META-INF/cxs/mappings/session.json | 22 ++++++++--
13 files changed, 215 insertions(+), 88 deletions(-)
diff --git
a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index d85ede2..f1949ed 100644
---
a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++
b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -349,21 +349,11 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
}
}
if (!indexExists) {
- logger.info("{} index doesn't exist yet, creating it...",
indexName);
- Map<String, Object> indexMappings = new HashMap<>();
- indexMappings.put("_default_", mappings.get("_default_"));
- for (Map.Entry<String, String> entry :
mappings.entrySet()) {
- if (!itemsMonthlyIndexed.contains(entry.getKey()) &&
!indexNames.containsKey(entry.getKey())) {
- indexMappings.put(entry.getKey(),
entry.getValue());
- }
- }
-
- internalCreateIndex(indexName, indexMappings);
+ logger.info("{} index doesn't exist yet, creating it...",
getIndex("profile", null));
+ internalCreateIndex(getIndex("profile", null),
mappings.get("profile"));
} else {
- logger.info("Found index {}, ElasticSearch started
successfully.", indexName);
- for (Map.Entry<String, String> entry :
mappings.entrySet()) {
- createMapping(entry.getKey(), entry.getValue());
- }
+ logger.info("Found index {}, ElasticSearch started
successfully.", getIndex("profile", null));
+ createMapping(getIndex("profile", null),
mappings.get("profile"));
}
createMonthlyIndexTemplate();
@@ -915,8 +905,8 @@ public class ElasticSearchPersistenceServiceImpl implements
PersistenceService,
public boolean createMonthlyIndexTemplate() {
Boolean result = new InClassLoaderExecute<Boolean>(metricsService,
this.getClass().getName() + ".createMonthlyIndexTemplate") {
protected Boolean execute(Object... args) throws IOException {
- PutIndexTemplateRequest putIndexTemplateRequest = new
PutIndexTemplateRequest("context-monthly-indices")
- .patterns(Arrays.asList(indexName + "*" +
INDEX_DATE_PREFIX + "*"))
+ PutIndexTemplateRequest putIndexTemplateRequest = new
PutIndexTemplateRequest("context-event-monthly-indices")
+ .patterns(Arrays.asList(indexName + "-event-" +
INDEX_DATE_PREFIX + "*"))
.settings("{\n" +
" \"index\" : {\n" +
" \"number_of_shards\" : " +
monthlyIndexNumberOfShards + ",\n" +
@@ -932,14 +922,7 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
" }\n" +
" }\n" +
"}\n", XContentType.JSON);
- Map<String, Object> indexMappings = new HashMap<String,
Object>();
- indexMappings.put("_default_", mappings.get("_default_"));
- for (Map.Entry<String, String> entry : mappings.entrySet()) {
- if (itemsMonthlyIndexed.contains(entry.getKey())) {
- indexMappings.put(entry.getKey(), entry.getValue());
- }
- }
- putIndexTemplateRequest.mapping(indexMappings);
+ putIndexTemplateRequest.mapping(mappings.get("event"),
XContentType.JSON);
AcknowledgedResponse putIndexTemplateResponse =
client.indices().putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT);
return putIndexTemplateResponse.isAcknowledged();
}
@@ -957,14 +940,7 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
GetIndexRequest getIndexRequest = new
GetIndexRequest(indexName);
boolean indexExists = client.indices().exists(getIndexRequest,
RequestOptions.DEFAULT);
if (!indexExists) {
- Map<String, Object> indexMappings = new HashMap<String,
Object>();
- indexMappings.put("_default_", mappings.get("_default_"));
- for (Map.Entry<String, String> entry :
mappings.entrySet()) {
- if (indexNames.containsKey(entry.getKey()) &&
indexNames.get(entry.getKey()).equals(indexName)) {
- indexMappings.put(entry.getKey(),
entry.getValue());
- }
- }
- internalCreateIndex(indexName, indexMappings);
+ internalCreateIndex(indexName, mappings.get(indexName));
}
return !indexExists;
}
@@ -995,7 +971,7 @@ public class ElasticSearchPersistenceServiceImpl implements
PersistenceService,
}
}
- private void internalCreateIndex(String indexName, Map<String, Object>
mappings) throws IOException {
+ private void internalCreateIndex(String indexName, String mappingSource)
throws IOException {
CreateIndexRequest createIndexRequest = new
CreateIndexRequest(indexName);
createIndexRequest.settings("{\n" +
" \"index\" : {\n" +
@@ -1013,7 +989,7 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
" }\n" +
"}\n", XContentType.JSON);
- createIndexRequest.mapping(mappings);
+ createIndexRequest.mapping(mappingSource, XContentType.JSON);
client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
}
@@ -1026,9 +1002,6 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
@Override
public void createMapping(String type, String source) {
- if (type.equals("_default_")) {
- return;
- }
try {
if (itemsMonthlyIndexed.contains(type)) {
createMonthlyIndexTemplate();
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json
deleted file mode 100644
index 9bc4ab0..0000000
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "_default_": {
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ]
- }
-}
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
index 3e36a7f..6290fb6 100644
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
+++
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
@@ -1,5 +1,22 @@
{
- "campaign": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
"properties": {
"cost": {
"type": "double"
@@ -27,5 +44,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
index 9ab8779..9b05a7f 100644
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
+++
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
@@ -1,6 +1,23 @@
{
- "campaignevent": {
- "properties": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
"cost": {
"type": "double"
},
@@ -24,5 +41,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
index 69c9f08..829091a 100644
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
+++
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json
@@ -1,10 +1,26 @@
{
- "event": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
"properties": {
"timeStamp": {
"type": "date"
}
}
- }
}
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
index 3e0bb6f..9627340 100644
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
+++
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
@@ -1,6 +1,23 @@
{
- "goal": {
- "properties": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
"metadata": {
"properties": {
@@ -19,5 +36,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
index ce3a8f3..901266d 100644
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
+++
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
@@ -1,6 +1,23 @@
{
- "personaSession": {
- "properties": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
"duration" : {
"type" : "long"
},
@@ -22,5 +39,4 @@
"type" : "long"
}
}
- }
}
\ No newline at end of file
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
index 29d40b7..2fef62f 100644
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
+++
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json
@@ -1,6 +1,23 @@
{
- "profile": {
- "properties": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
"properties": {
"properties": {
@@ -19,5 +36,4 @@
}
}
}
- }
}
\ No newline at end of file
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
index 99969f7..6adbd2a 100644
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
+++
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
@@ -1,6 +1,23 @@
{
- "propertyType": {
- "properties": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
"dateRanges": {
"properties": {
}
@@ -41,5 +58,4 @@
"type": "double"
}
}
- }
}
\ No newline at end of file
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
index ea6bb26..3561e61 100644
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
+++
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
@@ -1,6 +1,23 @@
{
- "rule": {
- "properties": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
"metadata": {
"properties": {
"enabled": {
@@ -27,5 +44,4 @@
"type": "boolean"
}
}
- }
}
\ No newline at end of file
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
index cda229b..8e256dd 100644
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
+++
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
@@ -1,5 +1,22 @@
{
- "scoring": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
"properties": {
"metadata": {
@@ -19,5 +36,4 @@
}
}
}
- }
}
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
index 5e20436..fa93d5b 100644
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
+++
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
@@ -1,5 +1,22 @@
{
- "segment": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
"properties": {
"metadata": {
@@ -20,5 +37,4 @@
}
}
}
- }
}
diff --git
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
index d687dce..7cc4061 100644
---
a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
+++
b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json
@@ -1,6 +1,23 @@
{
- "session": {
- "properties": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
"duration": {
"type": "long"
},
@@ -24,5 +41,4 @@
"type": "date"
}
}
- }
}
\ No newline at end of file
