Repository: incubator-unomi Updated Branches: refs/heads/feature-UNOMI-70-ES5X 603346ad3 -> 8b5a44bb1
UNOMI-70 : use folding + keyword mapping. detect field type in order to choose usage of keyword for sort and aggregates Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/8b5a44bb Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/8b5a44bb Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/8b5a44bb Branch: refs/heads/feature-UNOMI-70-ES5X Commit: 8b5a44bb1d9bc0090a6e4d11587d02b542acc567 Parents: 603346a Author: Thomas Draier <[email protected]> Authored: Tue Dec 20 11:04:32 2016 +0100 Committer: Thomas Draier <[email protected]> Committed: Tue Dec 20 11:04:32 2016 +0100 ---------------------------------------------------------------------- .../META-INF/cxs/mappings/geonameEntry.json | 48 +-------------- .../META-INF/cxs/mappings/userList.json | 15 ----- .../ElasticSearchPersistenceServiceImpl.java | 64 ++++++++++++++------ .../META-INF/cxs/mappings/_default_.json | 22 +++++++ .../META-INF/cxs/mappings/campaign.json | 24 -------- .../META-INF/cxs/mappings/campaignevent.json | 24 -------- .../resources/META-INF/cxs/mappings/event.json | 18 ------ .../resources/META-INF/cxs/mappings/goal.json | 19 +----- .../META-INF/cxs/mappings/personaSession.json | 20 +----- .../META-INF/cxs/mappings/profile.json | 14 +---- .../META-INF/cxs/mappings/propertyType.json | 45 -------------- .../resources/META-INF/cxs/mappings/rule.json | 12 ---- .../META-INF/cxs/mappings/scoring.json | 16 +---- .../META-INF/cxs/mappings/segment.json | 17 +----- .../META-INF/cxs/mappings/session.json | 25 +------- .../persistence/spi/PersistenceService.java | 10 +++ .../PropertyConditionESQueryBuilder.java | 4 +- .../conditions/PropertyConditionEvaluator.java | 12 +++- .../services/services/GoalsServiceImpl.java | 20 ++++-- .../services/services/QueryServiceImpl.java | 9 ++- 20 files changed, 124 insertions(+), 314 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json ---------------------------------------------------------------------- diff --git a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json index 612f3ca..64038ab 100644 --- a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json +++ b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json @@ -1,48 +1,10 @@ { "geonameEntry": { "properties": { - "admin1Code": { - "type": "keyword" - }, - "admin2Code": { - "type": "keyword" - }, - "admin3Code": { - "type": "keyword" - }, - "admin4Code": { - "type": "keyword" - }, - "alternatenames": { - "type": "keyword" - }, - "asciiname": { - "type": "keyword" - }, - "cc2": { - "type": "keyword" - }, - "countryCode": { - "type": "keyword" - }, - "dem": { - "type": "keyword" - }, "elevation": { "type": "long" }, - "featureClass": { - "type": "keyword" - }, - "featureCode": { - "type": "keyword" - }, - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, + "location": { "type": "geo_point" }, @@ -50,15 +12,11 @@ "type": "date", "format": "strict_date_optional_time||epoch_millis" }, - "name": { - "type": "keyword" - }, + "population": { "type": "long" - }, - "timezone": { - "type": "keyword" } + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json ---------------------------------------------------------------------- diff --git a/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json b/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json index 53fd043..d73e8b3 100644 --- a/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json +++ b/extensions/lists-extension/services/src/main/resources/META-INF/cxs/mappings/userList.json @@ -1,12 +1,6 @@ { "userList": { "properties": { - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, "metadata": { "properties": { "enabled": { @@ -15,20 +9,11 @@ "hidden": { "type": "boolean" }, - "id": { - "type": "keyword" - }, "missingPlugins": { "type": "boolean" }, "readOnly": { "type": "boolean" - }, - "scope": { - "type": "keyword" - }, - "tags": { - "type": "keyword" } } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java index 5e806d1..ee85d98 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java @@ -33,10 +33,7 @@ import org.apache.unomi.api.query.DateRange; import org.apache.unomi.api.query.IpRange; import org.apache.unomi.api.query.NumericRange; import org.apache.unomi.api.services.ClusterService; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; +import org.apache.unomi.persistence.elasticsearch.conditions.*; import org.apache.unomi.persistence.spi.CustomObjectMapper; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.persistence.spi.aggregate.*; @@ -175,6 +172,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, private String bulkProcessorFlushInterval = "5s"; private String bulkProcessorBackoffPolicy = "exponential"; + private Map<String, Map<String, Map<String, Object>>> knownMappings = new HashMap<>(); + public void setBundleContext(BundleContext bundleContext) { this.bundleContext = bundleContext; } @@ -388,6 +387,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, if (!indexExists) { logger.info("{} index doesn't exist yet, creating it...", indexName); Map<String,String> indexMappings = new HashMap<String,String>(); + indexMappings.put("_default_",mappings.get("_default_")); for (Map.Entry<String, String> entry : mappings.entrySet()) { if (!itemsMonthlyIndexed.contains(entry.getKey()) && !indexNames.containsKey(entry.getKey())) { indexMappings.put(entry.getKey(), entry.getValue()); @@ -631,6 +631,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, logger.info("{} index doesn't exist yet, creating it...", monthlyIndexName); Map<String,String> indexMappings = new HashMap<String,String>(); + indexMappings.put("_default_",mappings.get("_default_")); for (Map.Entry<String, String> entry : mappings.entrySet()) { if (itemsMonthlyIndexed.contains(entry.getKey())) { indexMappings.put(entry.getKey(), entry.getValue()); @@ -928,6 +929,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, boolean indexExists = indicesExistsResponse.isExists(); if (!indexExists) { Map<String,String> indexMappings = new HashMap<String,String>(); + indexMappings.put("_default_",mappings.get("_default_")); for (Map.Entry<String, String> entry : mappings.entrySet()) { if (indexNames.containsKey(entry.getKey()) && indexNames.get(entry.getKey()).equals(indexName)) { indexMappings.put(entry.getKey(), entry.getValue()); @@ -958,17 +960,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, CreateIndexRequestBuilder builder = client.admin().indices().prepareCreate(indexName) .setSettings("{\n" + " \"analysis\": {\n" + - " \"tokenizer\": {\n" + - " \"myTokenizer\": {\n" + - " \"type\":\"pattern\",\n" + - " \"pattern\":\".*\",\n" + - " \"group\":0\n" + - " }\n" + - " },\n" + " \"analyzer\": {\n" + " \"folding\": {\n" + " \"type\":\"custom\",\n" + - " \"tokenizer\": \"myTokenizer\",\n" + + " \"tokenizer\": \"keyword\",\n" + " \"filter\": [ \"lowercase\", \"asciifolding\" ]\n" + " }\n" + " }\n" + @@ -995,6 +990,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, @Override public void createMapping(String type, String source) { + if (type.equals("_default_")) { + return; + } if (itemsMonthlyIndexed.contains(type)) { createMapping(type, source, indexName + "-*"); } else if (indexNames.containsKey(type)) { @@ -1042,6 +1040,36 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, }.executeInClassLoader(); } + public Map<String, Object> getPropertyMapping(String property, String itemType) { + Map<String, Map<String, Object>> mappings = knownMappings.get(itemType); + Map<String, Object> result = getPropertyMapping(property, mappings); + if (result == null) { + mappings = getPropertiesMapping(itemType); + knownMappings.put(itemType, mappings); + result = getPropertyMapping(property, mappings); + } + return result; + } + + private Map<String, Object> getPropertyMapping(String property, Map<String, Map<String, Object>> mappings) { + Map<String, Object> propMapping = null; + String[] properties = StringUtils.split(property, '.'); + for (int i = 0; i < properties.length; i++) { + String s = properties[i]; + if (mappings != null) { + propMapping = mappings.get(s); + if (i == properties.length - 1) { + return propMapping; + } else { + mappings = (Map<String, Map<String, Object>>) propMapping.get("properties"); + } + } else { + return null; + } + } + return propMapping; + } + public boolean saveQuery(final String queryName, final String query) { return new InClassLoaderExecute<Boolean>() { protected Boolean execute(Object... args) { @@ -1138,12 +1166,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, @Override public <T extends Item> List<T> query(final String fieldName, final String[] fieldValues, String sortBy, final Class<T> clazz) { - return query(QueryBuilders.termsQuery(fieldName, fieldValues), sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz), null).getList(); + return query(QueryBuilders.termsQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValues)), sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz), null).getList(); } @Override public <T extends Item> PartialList<T> query(String fieldName, String fieldValue, String sortBy, Class<T> clazz, int offset, int size) { - return query(QueryBuilders.termQuery(fieldName, fieldValue), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null); + return query(QueryBuilders.termQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValue)), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null); } @Override @@ -1247,9 +1275,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, } } else { String name = StringUtils.substringBeforeLast(sortByElement,":"); - if (name.equals("metadata.name") || name.equals("metadata.description") - || name.equals("properties.firstName") || name.equals("properties.lastName") - ) { + Map<String,Object> propertyMapping = getPropertyMapping(name,itemType); + if (propertyMapping != null + && "text".equals(propertyMapping.get("type")) + && propertyMapping.containsKey("fields") + && ((Map)propertyMapping.get("fields")).containsKey("keyword")) { name += ".keyword"; } if (sortByElement.endsWith(":desc")) { http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json new file mode 100644 index 0000000..9bc4ab0 --- /dev/null +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/_default_.json @@ -0,0 +1,22 @@ +{ + "_default_": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json index acd4b85..3e36a7f 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json @@ -4,27 +4,12 @@ "cost": { "type": "double" }, - "currency": { - "type": "keyword" - }, "startDate": { "type": "date" }, "endDate": { "type": "date" }, - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, - "primaryGoal": { - "type": "keyword" - }, - "timezone": { - "type": "keyword" - }, "metadata": { "properties": { "enabled": { @@ -33,20 +18,11 @@ "hidden": { "type": "boolean" }, - "id": { - "type": "keyword" - }, "missingPlugins": { "type": "boolean" }, "readOnly": { "type": "boolean" - }, - "scope": { - "type": "keyword" - }, - "tags": { - "type": "keyword" } } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json index 7308995..9ab8779 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json @@ -1,27 +1,12 @@ { "campaignevent": { "properties": { - "campaignId": { - "type": "keyword" - }, "cost": { "type": "double" }, - "currency": { - "type": "keyword" - }, "eventDate": { "type": "date" }, - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, - "timezone": { - "type": "keyword" - }, "metadata": { "properties": { "enabled": { @@ -30,20 +15,11 @@ "hidden": { "type": "boolean" }, - "id": { - "type": "keyword" - }, "missingPlugins": { "type": "boolean" }, "readOnly": { "type": "boolean" - }, - "scope": { - "type": "keyword" - }, - "tags": { - "type": "keyword" } } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json index c240b3b..69c9f08 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json @@ -1,24 +1,6 @@ { "event": { "properties": { - "eventType": { - "type": "keyword" - }, - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, - "profileId": { - "type": "keyword" - }, - "scope": { - "type": "keyword" - }, - "sessionId": { - "type": "keyword" - }, "timeStamp": { "type": "date" } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json index 081915b..3e0bb6f 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json @@ -1,15 +1,7 @@ { "goal": { "properties": { - "campaignId": { - "type": "keyword" - }, - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, + "metadata": { "properties": { "enabled": { @@ -18,20 +10,11 @@ "hidden": { "type": "boolean" }, - "id": { - "type": "keyword" - }, "missingPlugins": { "type": "boolean" }, "readOnly": { "type": "boolean" - }, - "scope": { - "type": "keyword" - }, - "tags": { - "type": "keyword" } } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json index cb26f1e..ce3a8f3 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json @@ -4,31 +4,13 @@ "duration" : { "type" : "long" }, - "itemId" : { - "type" : "keyword" - }, - "itemType" : { - "type" : "keyword" - }, + "timeStamp": { "type": "date" }, "lastEventDate": { "type": "date" }, - "profile" : { - "properties" : { - "itemId" : { - "type" : "keyword" - }, - "itemType" : { - "type" : "keyword" - } - } - }, - "profileId" : { - "type" : "keyword" - }, "properties" : { "properties" : { "location" : { http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json index 665941d..29d40b7 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json @@ -1,12 +1,7 @@ { "profile": { "properties": { - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, + "properties": { "properties": { "firstVisit": { @@ -22,13 +17,6 @@ "type": "long" } } - }, - "systemProperties" : { - "properties" : { - "mergeIdentifier" : { - "type" : "keyword" - } - } } } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json index bdba719..99969f7 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json @@ -1,34 +1,10 @@ { "propertyType": { "properties": { - "automaticMappingsFrom": { - "type": "keyword" - }, "dateRanges": { "properties": { - "from": { - "type": "keyword" - }, - "key": { - "type": "keyword" - }, - "to": { - "type": "keyword" - } } }, - "defaultValue": { - "type": "keyword" - }, - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, - "mergeStrategy": { - "type": "keyword" - }, "metadata": { "properties": { "enabled": { @@ -37,20 +13,11 @@ "hidden": { "type": "boolean" }, - "id": { - "type": "keyword" - }, "missingPlugins": { "type": "boolean" }, "readOnly": { "type": "boolean" - }, - "scope": { - "type": "keyword" - }, - "tags": { - "type": "keyword" } } }, @@ -62,9 +29,6 @@ "from": { "type": "double" }, - "key": { - "type": "keyword" - }, "to": { "type": "double" } @@ -75,15 +39,6 @@ }, "rank": { "type": "double" - }, - "tags": { - "type": "keyword" - }, - "target": { - "type": "keyword" - }, - "type": { - "type": "keyword" } } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json index aad6029..ea6bb26 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json @@ -1,12 +1,6 @@ { "rule": { "properties": { - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, "metadata": { "properties": { "enabled": { @@ -15,17 +9,11 @@ "hidden": { "type": "boolean" }, - "id": { - "type": "keyword" - }, "missingPlugins": { "type": "boolean" }, "readOnly": { "type": "boolean" - }, - "scope": { - "type": "keyword" } } }, http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json index b46d7e2..cda229b 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json @@ -1,12 +1,7 @@ { "scoring": { "properties": { - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, + "metadata": { "properties": { "enabled": { @@ -15,20 +10,11 @@ "hidden": { "type": "boolean" }, - "id": { - "type": "keyword" - }, "missingPlugins": { "type": "boolean" }, "readOnly": { "type": "boolean" - }, - "scope": { - "type": "keyword" - }, - "tags": { - "type": "keyword" } } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json index 322ff19..5e20436 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json @@ -1,12 +1,7 @@ { "segment": { "properties": { - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, + "metadata": { "properties": { "enabled": { @@ -15,20 +10,12 @@ "hidden": { "type": "boolean" }, - "id": { - "type": "keyword" - }, + "missingPlugins": { "type": "boolean" }, "readOnly": { "type": "boolean" - }, - "scope": { - "type": "keyword" - }, - "tags": { - "type": "keyword" } } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json index b1759a3..d687dce 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json @@ -4,28 +4,11 @@ "duration": { "type": "long" }, - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - }, + "lastEventDate": { "type": "date" }, - "profileId": { - "type": "keyword" - }, - "profile": { - "properties": { - "itemId": { - "type": "keyword" - }, - "itemType": { - "type": "keyword" - } - } - }, + "properties": { "properties": { "location": { @@ -33,9 +16,7 @@ } } }, - "scope": { - "type": "keyword" - }, + "size": { "type": "long" }, http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java ---------------------------------------------------------------------- diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java index e191829..132c583 100644 --- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java @@ -174,6 +174,16 @@ public interface PersistenceService { Map<String, Map<String, Object>> getPropertiesMapping(String itemType); /** + * Retrieve the mapping for one specific property for a given type. + * + * @param property the property name (can use nested dot notation) + * @param itemType the itemType we want to retrieve the mappings for + * @return + */ + Map<String, Object> getPropertyMapping(String property, String itemType); + + + /** * Create mapping * @param type * @param source http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java ---------------------------------------------------------------------- diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java index 31aa0a3..c36722b 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java @@ -44,12 +44,12 @@ public class PropertyConditionESQueryBuilder implements ConditionESQueryBuilder throw new IllegalArgumentException("Impossible to build ES filter, condition is not valid, comparisonOperator and propertyName properties should be provided"); } - String expectedValue = (String) condition.getParameter("propertyValue"); + String expectedValue = ConditionContextHelper.foldToASCII((String) condition.getParameter("propertyValue")); Object expectedValueInteger = condition.getParameter("propertyValueInteger"); Object expectedValueDate = condition.getParameter("propertyValueDate"); Object expectedValueDateExpr = condition.getParameter("propertyValueDateExpr"); - List<?> expectedValues = (List<?>) condition.getParameter("propertyValues"); + List<?> expectedValues = ConditionContextHelper.foldToASCII((List<?>) condition.getParameter("propertyValues")); List<?> expectedValuesInteger = (List<?>) condition.getParameter("propertyValuesInteger"); List<?> expectedValuesDate = (List<?>) condition.getParameter("propertyValuesDate"); List<?> expectedValuesDateExpr = (List<?>) condition.getParameter("propertyValuesDateExpr"); http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java ---------------------------------------------------------------------- diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java index d8b7754..68e58a6 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java @@ -83,7 +83,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator { return false; } - List<Object> actual = getValueSet(actualValue); + List<Object> actual = ConditionContextHelper.foldToASCII(getValueSet(actualValue)); boolean result = true; @@ -136,7 +136,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator { String op = (String) condition.getParameter("comparisonOperator"); String name = (String) condition.getParameter("propertyName"); - String expectedValue = (String) condition.getParameter("propertyValue"); + String expectedValue = ConditionContextHelper.foldToASCII((String) condition.getParameter("propertyValue")); Object expectedValueInteger = condition.getParameter("propertyValueInteger"); Object expectedValueDate = condition.getParameter("propertyValueDate"); Object expectedValueDateExpr = condition.getParameter("propertyValueDateExpr"); @@ -165,6 +165,9 @@ public class PropertyConditionEvaluator implements ConditionEvaluator { actualValue = null; } } + if (actualValue instanceof String) { + actualValue = ConditionContextHelper.foldToASCII((String) actualValue); + } if(op == null) { return false; @@ -175,6 +178,9 @@ public class PropertyConditionEvaluator implements ConditionEvaluator { } else if (op.equals("equals")) { if (actualValue instanceof Collection) { for (Object o : ((Collection<?>)actualValue)) { + if (o instanceof String) { + o = ConditionContextHelper.foldToASCII((String) o); + } if (compare(o, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr) == 0) { return true; } @@ -214,7 +220,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator { } else if (op.equals("matchesRegex")) { return expectedValue != null && Pattern.compile(expectedValue).matcher(actualValue.toString()).matches(); } else if (op.equals("in") || op.equals("notIn") || op.equals("hasSomeOf") || op.equals("hasNoneOf") || op.equals("all")) { - List<?> expectedValues = (List<?>) condition.getParameter("propertyValues"); + List<?> expectedValues = ConditionContextHelper.foldToASCII((List<?>) condition.getParameter("propertyValues")); List<?> expectedValuesInteger = (List<?>) condition.getParameter("propertyValuesInteger"); List<?> expectedValuesDate = (List<?>) condition.getParameter("propertyValuesDate"); List<?> expectedValuesDateExpr = (List<?>) condition.getParameter("propertyValuesDateExpr"); http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java ---------------------------------------------------------------------- diff --git a/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java index a99f155..a6bd1d6 100644 --- a/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/services/GoalsServiceImpl.java @@ -474,25 +474,33 @@ public class GoalsServiceImpl implements GoalsService, SynchronousBundleListener // resolve aggregate BaseAggregate aggregate = null; - if(query != null && query.getAggregate() != null && query.getAggregate().getProperty() != null) { + String property = query.getAggregate().getProperty(); + if(query != null && query.getAggregate() != null && property != null) { if (query.getAggregate().getType() != null){ // try to guess the aggregate type if(query.getAggregate().getType().equals("date")) { String interval = (String) query.getAggregate().getParameters().get("interval"); String format = (String) query.getAggregate().getParameters().get("format"); - aggregate = new DateAggregate(query.getAggregate().getProperty(), interval, format); + aggregate = new DateAggregate(property, interval, format); } else if (query.getAggregate().getType().equals("dateRange") && query.getAggregate().getDateRanges() != null && query.getAggregate().getDateRanges().size() > 0) { String format = (String) query.getAggregate().getParameters().get("format"); - aggregate = new DateRangeAggregate(query.getAggregate().getProperty(), format, query.getAggregate().getDateRanges()); + aggregate = new DateRangeAggregate(property, format, query.getAggregate().getDateRanges()); } else if (query.getAggregate().getType().equals("numericRange") && query.getAggregate().getNumericRanges() != null && query.getAggregate().getNumericRanges().size() > 0) { - aggregate = new NumericRangeAggregate(query.getAggregate().getProperty(), query.getAggregate().getNumericRanges()); + aggregate = new NumericRangeAggregate(property, query.getAggregate().getNumericRanges()); } else if (query.getAggregate().getType().equals("ipRange") && query.getAggregate().ipRanges() != null && query.getAggregate().ipRanges().size() > 0) { - aggregate = new IpRangeAggregate(query.getAggregate().getProperty(), query.getAggregate().ipRanges()); + aggregate = new IpRangeAggregate(property, query.getAggregate().ipRanges()); } } if(aggregate == null){ - aggregate = new TermsAggregate(query.getAggregate().getProperty() + ".keyword"); + Map<String,Object> propertyMapping = persistenceService.getPropertyMapping(property, Session.ITEM_TYPE); + if (propertyMapping != null + && "text".equals(propertyMapping.get("type")) + && propertyMapping.containsKey("fields") + && ((Map)propertyMapping.get("fields")).containsKey("keyword")) { + property += ".keyword"; + } + aggregate = new TermsAggregate(property); } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/8b5a44bb/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java ---------------------------------------------------------------------- diff --git a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java index 79edd43..30e6dab 100644 --- a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java @@ -83,7 +83,14 @@ public class QueryServiceImpl implements QueryService { } // fall back on terms aggregate - return persistenceService.aggregateQuery(query.getCondition(), new TermsAggregate(property + ".keyword"), itemType); + Map<String,Object> propertyMapping = persistenceService.getPropertyMapping( property, itemType); + if (propertyMapping != null + && "text".equals(propertyMapping.get("type")) + && propertyMapping.containsKey("fields") + && ((Map)propertyMapping.get("fields")).containsKey("keyword")) { + property += ".keyword"; + } + return persistenceService.aggregateQuery(query.getCondition(), new TermsAggregate(property), itemType); } return getAggregate(itemType, property);
