UNOMI-28 Upgrade to ElasticSearch 2.x Used this resource to perform changes: https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking_20_java_api_changes.html
A lot of changes are contained in this commit, notably : - Moved data directory from KARAF_HOME/data to KARAF_HOME/elasticsearch/data - Had to build a JSON parser to rewrite field names with "." to escape them with "__DOT__" - Fixes on dates - Replace all filter classes with query classes as these don't exist in ES 2.x any more - Replace and/or ES clauses with bool must/should - Had to replace deleteByQuery support to add plugin by default, since in ES 2.x they externalized the functionality - Had to install Groovy and Expression modules since they are no longer part of the core What is NOT done by this patch is the migration procedure Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/131a06aa Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/131a06aa Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/131a06aa Branch: refs/heads/UNOMI-28-ES-2-X-UPGRADE Commit: 131a06aa78fc86223359c11c7a40a4e653dab434 Parents: 1c5d74a Author: Serge Huber <[email protected]> Authored: Tue Aug 9 15:55:03 2016 +0200 Committer: Serge Huber <[email protected]> Committed: Tue Aug 9 15:55:03 2016 +0200 ---------------------------------------------------------------------- .../META-INF/cxs/mappings/geonameEntry.json | 2 +- itests/pom.xml | 5 +- .../java/org/apache/unomi/itests/AllTests.java | 11 +- .../apache/unomi/itests/ProfileServiceTest.java | 53 +++ kar/src/main/feature/feature.xml | 12 +- package/pom.xml | 54 ++- persistence-elasticsearch/core/pom.xml | 207 +++------- .../ElasticSearchPersistenceServiceImpl.java | 165 +++++--- .../elasticsearch/FieldDotEscaper.java | 108 ++++++ .../elasticsearch/FieldDotJsonTransformer.java | 290 ++++++++++++++ .../conditions/ConditionContextHelper.java | 4 +- .../conditions/ConditionESQueryBuilder.java | 4 +- .../ConditionESQueryBuilderDispatcher.java | 16 +- .../META-INF/cxs/mappings/campaign.json | 4 +- .../META-INF/cxs/mappings/campaignevent.json | 2 +- .../resources/META-INF/cxs/mappings/event.json | 2 +- .../META-INF/cxs/mappings/profile.json | 2 +- .../META-INF/cxs/mappings/session.json | 4 +- .../core/src/main/resources/elasticsearch.yml | 381 +++---------------- .../elasticsearch/FieldDotEscapeTest.java | 95 +++++ .../FieldDotJsonTransformerTest.java | 61 +++ .../core/src/test/resources/complex.json | 50 +++ .../core/src/test/resources/profile.json | 1 + .../plugin/security/SecurityPlugin.java | 14 +- .../plugin/security/SecurityPluginModule.java | 3 +- .../plugin/security/SecurityPluginService.java | 4 +- .../BooleanConditionESQueryBuilder.java | 20 +- ...onByPointSessionConditionESQueryBuilder.java | 10 +- .../MatchAllConditionESQueryBuilder.java | 8 +- .../conditions/NotConditionESQueryBuilder.java | 8 +- .../PastEventConditionESQueryBuilder.java | 8 +- .../PropertyConditionESQueryBuilder.java | 51 +-- .../conditions/PropertyConditionEvaluator.java | 3 +- ...rceEventPropertyConditionESQueryBuilder.java | 27 +- .../HoverEventConditionESQueryBuilder.java | 23 +- pom.xml | 8 +- 36 files changed, 1044 insertions(+), 676 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json ---------------------------------------------------------------------- diff --git a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json index 6ccf39b..63664fe 100644 --- a/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json +++ b/extensions/geonames/services/src/main/resources/META-INF/cxs/mappings/geonameEntry.json @@ -58,7 +58,7 @@ }, "modificationDate": { "type": "date", - "format": "dateOptionalTime" + "format": "strict_date_optional_time||epoch_millis" }, "name": { "type": "string", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/itests/pom.xml ---------------------------------------------------------------------- diff --git a/itests/pom.xml b/itests/pom.xml index b156a45..9c2c00b 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -106,7 +106,10 @@ <artifactId>javax.inject</artifactId> <scope>test</scope> </dependency> - + <dependency> + <groupId>org.osgi</groupId> + <artifactId>org.osgi.core</artifactId> + </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/itests/src/test/java/org/apache/unomi/itests/AllTests.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/AllTests.java b/itests/src/test/java/org/apache/unomi/itests/AllTests.java index 8f017ba..0fbd383 100644 --- a/itests/src/test/java/org/apache/unomi/itests/AllTests.java +++ b/itests/src/test/java/org/apache/unomi/itests/AllTests.java @@ -27,11 +27,12 @@ import org.junit.runners.Suite.SuiteClasses; * @author Sergiy Shyrkov */ @RunWith(Suite.class) -@SuiteClasses({ - //BasicTest.class, - ConditionEvaluatorTest.class, - ConditionESQueryBuilderTest.class, - SegmentTest.class +@SuiteClasses({ + //BasicTest.class, + ConditionEvaluatorTest.class, + ConditionESQueryBuilderTest.class, + SegmentTest.class, + ProfileServiceTest.class }) public class AllTests { } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java new file mode 100644 index 0000000..4f5dba8 --- /dev/null +++ b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package org.apache.unomi.itests; + +import org.apache.unomi.api.Profile; +import org.apache.unomi.api.services.ProfileService; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.ops4j.pax.exam.junit.PaxExam; +import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy; +import org.ops4j.pax.exam.spi.reactors.PerSuite; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; + +/** + * An integration test for the profile service + */ +@RunWith(PaxExam.class) +@ExamReactorStrategy(PerSuite.class) +public class ProfileServiceTest extends BaseTest { + + private final static Logger LOGGER = LoggerFactory.getLogger(SegmentTest.class); + private final static String TEST_PROFILE_ID = "test-profile-id"; + @Inject + protected ProfileService profileService; + + @Test + public void testProfileDelete() { + Profile profile = new Profile(); + profile.setItemId(TEST_PROFILE_ID); + profileService.save(profile); + LOGGER.info("Profile saved, now testing profile delete..."); + profileService.delete(TEST_PROFILE_ID, false); + LOGGER.info("Profile deleted successfully."); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/kar/src/main/feature/feature.xml ---------------------------------------------------------------------- diff --git a/kar/src/main/feature/feature.xml b/kar/src/main/feature/feature.xml index 5e63b8b..0a66af8 100644 --- a/kar/src/main/feature/feature.xml +++ b/kar/src/main/feature/feature.xml @@ -31,12 +31,12 @@ <configfile finalname="/etc/org.apache.unomi.geonames.cfg">mvn:org.apache.unomi/cxs-geonames-services/${project.version}/cfg/geonamescfg</configfile> <configfile finalname="/etc/elasticsearch.yml">mvn:org.apache.unomi/unomi-persistence-elasticsearch-core/${project.version}/yml/elasticsearchconfig</configfile> <bundle start-level="75">mvn:commons-io/commons-io/2.4</bundle> - <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-core/2.4.0</bundle> - <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-databind/2.4.0</bundle> - <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-annotations/2.4.0</bundle> - <bundle start-level="75">mvn:com.fasterxml.jackson.jaxrs/jackson-jaxrs-base/2.4.0</bundle> - <bundle start-level="75">mvn:com.fasterxml.jackson.module/jackson-module-jaxb-annotations/2.4.0</bundle> - <bundle start-level="75">mvn:com.fasterxml.jackson.jaxrs/jackson-jaxrs-json-provider/2.4.0</bundle> + <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-core/${version.jackson.core}</bundle> + <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-databind/${version.jackson.core}</bundle> + <bundle start-level="75">mvn:com.fasterxml.jackson.core/jackson-annotations/${version.jackson.core}</bundle> + <bundle start-level="75">mvn:com.fasterxml.jackson.jaxrs/jackson-jaxrs-base/${version.jackson.core}</bundle> + <bundle start-level="75">mvn:com.fasterxml.jackson.module/jackson-module-jaxb-annotations/${version.jackson.jaxb}</bundle> + <bundle start-level="75">mvn:com.fasterxml.jackson.jaxrs/jackson-jaxrs-json-provider/${version.jackson.core}</bundle> <bundle start-level="75">mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.jakarta-regexp/1.4_1</bundle> <bundle start-level="75">mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.joda-time/2.3_1</bundle> <bundle start-level="75">mvn:org.apache.unomi/unomi-api/${project.version}</bundle> http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/package/pom.xml ---------------------------------------------------------------------- diff --git a/package/pom.xml b/package/pom.xml index eb6546b..c55aed0 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -122,7 +122,7 @@ <artifactId>maven-dependency-plugin</artifactId> <executions> <execution> - <id>unpack-sigar</id> + <id>unpack-deploy-by-query-plugin</id> <phase>generate-resources</phase> <goals> <goal>unpack</goal> @@ -130,12 +130,32 @@ <configuration> <artifactItems> <artifactItem> - <groupId>org.elasticsearch</groupId> - <artifactId>elasticsearch</artifactId> + <groupId>org.elasticsearch.plugin</groupId> + <artifactId>delete-by-query</artifactId> <version>${elasticsearch.version}</version> - <type>tar.gz</type> - <includes>**/lib/sigar/**</includes> - <outputDirectory>${project.build.directory}/assembly</outputDirectory> + <type>zip</type> + <outputDirectory>${project.build.directory}/assembly/elasticsearch/plugins/delete-by-query</outputDirectory> + </artifactItem> + <artifactItem> + <groupId>org.elasticsearch.module</groupId> + <artifactId>lang-groovy</artifactId> + <version>${elasticsearch.version}</version> + <type>zip</type> + <outputDirectory>${project.build.directory}/assembly/elasticsearch/modules/lang-groovy</outputDirectory> + </artifactItem> + <artifactItem> + <groupId>org.elasticsearch.module</groupId> + <artifactId>lang-expression</artifactId> + <version>${elasticsearch.version}</version> + <type>zip</type> + <outputDirectory>${project.build.directory}/assembly/elasticsearch/modules/lang-expression</outputDirectory> + </artifactItem> + <artifactItem> + <groupId>org.elasticsearch.module</groupId> + <artifactId>reindex</artifactId> + <version>${elasticsearch.version}</version> + <type>zip</type> + <outputDirectory>${project.build.directory}/assembly/elasticsearch/modules/reindex</outputDirectory> </artifactItem> </artifactItems> </configuration> @@ -210,28 +230,6 @@ </executions> </plugin> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <id>copy-karaf</id> - <phase>generate-resources</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <target> - <move todir="${project.build.directory}/assembly/lib/sigar"> - <fileset dir="${project.build.directory}/assembly/elasticsearch-${elasticsearch.version}/lib/sigar" /> - </move> - <delete dir="${project.build.directory}/assembly/elasticsearch-${elasticsearch.version}" /> - </target> - </configuration> - </execution> - </executions> - </plugin> - - <plugin> <groupId>org.apache.karaf.tooling</groupId> <artifactId>karaf-maven-plugin</artifactId> <executions> http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/pom.xml ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml index eddadf1..e44bd7a 100644 --- a/persistence-elasticsearch/core/pom.xml +++ b/persistence-elasticsearch/core/pom.xml @@ -52,101 +52,64 @@ <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> <version>${elasticsearch.version}</version> - </dependency> - - <!-- The following are optional dependencies from the ElasticSearch that are made mandatory --> - <dependency> - <groupId>org.apache.lucene</groupId> - <artifactId>lucene-expressions</artifactId> - <version>4.10.2</version> - </dependency> - <dependency> - <groupId>org.antlr</groupId> - <artifactId>antlr-runtime</artifactId> - <version>3.5</version> <scope>compile</scope> - <exclusions> - <exclusion> - <artifactId>stringtemplate</artifactId> - <groupId>org.antlr</groupId> - </exclusion> - </exclusions> </dependency> <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm</artifactId> - <version>4.1</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm-commons</artifactId> - <version>4.1</version> + <groupId>org.elasticsearch.plugin</groupId> + <artifactId>delete-by-query</artifactId> + <version>${elasticsearch.version}</version> <scope>compile</scope> - <exclusions> - <exclusion> - <artifactId>asm-tree</artifactId> - <groupId>org.ow2.asm</groupId> - </exclusion> - </exclusions> </dependency> + + <!-- The following are optional dependencies from the ElasticSearch that are made mandatory --> + <dependency> - <groupId>com.spatial4j</groupId> - <artifactId>spatial4j</artifactId> - <version>0.4.1</version> - </dependency> - <!--dependency> - <groupId>com.vividsolutions</groupId> - <artifactId>jts</artifactId> - <version>1.13</version> - </dependency--> - <dependency> - <groupId>org.codehaus.groovy</groupId> - <artifactId>groovy-all</artifactId> - <version>2.3.2</version> - </dependency> - <!-- - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - <version>1.2.17</version> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <scope>provided</scope> </dependency> - --> + <dependency> <groupId>org.slf4j</groupId> <artifactId>log4j-over-slf4j</artifactId> <version>1.7.7</version> + <scope>compile</scope> </dependency> - <!-- - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - <version>1.6.2</version> - </dependency> - --> <dependency> <groupId>net.java.dev.jna</groupId> <artifactId>jna</artifactId> <version>4.1.0</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.github.spullara.mustache.java</groupId> + <artifactId>compiler</artifactId> + <version>0.8.13</version> + <scope>compile</scope> </dependency> <dependency> - <groupId>org.fusesource</groupId> - <artifactId>sigar</artifactId> - <version>1.6.4</version> + <groupId>org.scala-lang</groupId> + <artifactId>scala-library</artifactId> + <version>2.8.1</version> + <scope>compile</scope> </dependency> <dependency> - <groupId>org.fusesource</groupId> - <artifactId>sigar</artifactId> - <version>1.6.4</version> - <classifier>native</classifier> + <groupId>com.twitter</groupId> + <artifactId>util-core</artifactId> + <version>4.0.0</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.noggit</groupId> + <artifactId>noggit</artifactId> + <version>0.7</version> + <scope>compile</scope> </dependency> - <!-- End of optional ElasticSearch dependencies --> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> - <scope>provided</scope> </dependency> <dependency> @@ -161,6 +124,18 @@ <groupId>org.mvel</groupId> <artifactId>mvel2</artifactId> </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <scope>test</scope> + </dependency> + </dependencies> <build> @@ -173,104 +148,30 @@ <instructions> <Import-Package> com.google.protobuf;resolution:=optional, - com.ibm.uvm.tools;resolution:=optional, - com.sun.jdmk.comm;resolution:=optional, - com.thoughtworks.qdox;resolution:=optional, - com.thoughtworks.qdox.model;resolution:=optional, - com.thoughtworks.xstream;resolution:=optional, com.vividsolutions.jts.*;resolution:=optional, - groovyjarjarasm.asm.tree.analysis;resolution:=optional, - javax.accessibility;resolution:=optional, javax.annotation;resolution:=optional, - javax.jmdns;resolution:=optional, - javax.jms;resolution:=optional, - javax.mail;resolution:=optional, - javax.mail.internet;resolution:=optional, - javax.management;resolution:=optional, - javax.management.modelmbean;resolution:=optional, - javax.management.remote;resolution:=optional, - javax.management.remote.rmi;resolution:=optional, - javax.management.timer;resolution:=optional, - javax.naming;resolution:=optional, + javax.crypto;resolution:=optional, + javax.crypto.spec;resolution:=optional, javax.net.ssl;resolution:=optional, - javax.print.attribute.standard;resolution:=optional, - javax.rmi.ssl;resolution:=optional, - javax.script;resolution:=optional, + javax.security.auth.x500;resolution:=optional, + javax.security.cert;resolution:=optional, javax.servlet;resolution:=optional, javax.servlet.http;resolution:=optional, - javax.servlet.jsp;resolution:=optional, - javax.sql;resolution:=optional, - javax.swing;resolution:=optional, - javax.swing.border;resolution:=optional, - javax.swing.event;resolution:=optional, - javax.swing.filechooser;resolution:=optional, - javax.swing.plaf;resolution:=optional, - javax.swing.plaf.metal;resolution:=optional, - javax.swing.table;resolution:=optional, - javax.swing.text;resolution:=optional, - javax.swing.text.html;resolution:=optional, - javax.swing.tree;resolution:=optional, - javax.swing.undo;resolution:=optional, - javax.xml.namespace;resolution:=optional, + javax.xml.bind;resolution:=optional, javax.xml.parsers;resolution:=optional, javax.xml.transform;resolution:=optional, javax.xml.transform.dom;resolution:=optional, javax.xml.transform.stream;resolution:=optional, - javax.xml.validation;resolution:=optional, - javax.xml.xpath;resolution:=optional, - jline;resolution:=optional, - jline.console;resolution:=optional, - jline.console.completer;resolution:=optional, - jline.console.history;resolution:=optional, - jline.internal;resolution:=optional, - junit.framework; resolution:=optional - junit.framework;resolution:=optional, - junit.textui; resolution:=optional - junit.textui;resolution:=optional, - org.antlr.stringtemplate;resolution:=optional, - org.apache.bsf;resolution:=optional, - org.apache.bsf.util;resolution:=optional, - org.apache.commons.cli;resolution:=optional, org.apache.commons.logging;resolution:=optional, - org.apache.ivy;resolution:=optional, - org.apache.ivy.core.cache;resolution:=optional, - org.apache.ivy.core.event;resolution:=optional, - org.apache.ivy.core.event.download;resolution:=optional, - org.apache.ivy.core.event.resolve;resolution:=optional, - org.apache.ivy.core.module.descriptor;resolution:=optional, - org.apache.ivy.core.module.id;resolution:=optional, - org.apache.ivy.core.report;resolution:=optional, - org.apache.ivy.core.resolve;resolution:=optional, - org.apache.ivy.core.settings;resolution:=optional, - org.apache.ivy.plugins.matcher;resolution:=optional, - org.apache.ivy.plugins.resolver;resolution:=optional, - org.apache.ivy.util;resolution:=optional, - org.apache.log;resolution:=optional, - org.apache.regexp; resolution:=optional org.apache.regexp;resolution:=optional, - org.apache.tomcat.jni; resolution:=optional, - org.apache.tools.ant;resolution:=optional, - org.apache.tools.ant.dispatch;resolution:=optional, - org.apache.tools.ant.helper;resolution:=optional, - org.apache.tools.ant.taskdefs; resolution:=optional - org.apache.tools.ant.taskdefs;resolution:=optional, - org.apache.tools.ant.types;resolution:=optional, - org.apache.tools.ant.util;resolution:=optional, - org.bouncycastle.asn1.x500;resolution:=optional, - org.bouncycastle.cert;resolution:=optional, - org.bouncycastle.cert.jcajce;resolution:=optional, - org.bouncycastle.jce.provider;resolution:=optional, - org.bouncycastle.operator;resolution:=optional, - org.bouncycastle.operator.jcajce;resolution:=optional, + org.apache.tomcat.jni;resolution:=optional, + org.bouncycastle.*;resolution:=optional, org.eclipse.jetty.npn;resolution:=optional, - org.fusesource.jansi;resolution:=optional, org.jboss.logging;resolution:=optional, org.jboss.marshalling;resolution:=optional, - org.junit;resolution:=optional, - org.objectweb.asm.tree;resolution:=optional, - sun.awt;resolution:=optional, - sun.java2d.pipe;resolution:=optional, - sun.misc; resolution:=optional, + org.jruby;resolution:=optional, + org.jruby.embed;resolution:=optional, + sun.misc;resolution:=optional, sun.security.util;resolution:=optional, sun.security.x509;resolution:=optional, * http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java index 7bf2f14..743def9 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java @@ -17,6 +17,7 @@ package org.apache.unomi.persistence.elasticsearch; +import com.google.common.collect.UnmodifiableIterator; import org.apache.unomi.api.ClusterNode; import org.apache.unomi.api.Item; import org.apache.unomi.api.PartialList; @@ -31,6 +32,7 @@ import org.apache.unomi.persistence.spi.CustomObjectMapper; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.persistence.spi.aggregate.*; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; @@ -40,26 +42,29 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.deletebyquery.DeleteByQueryAction; +import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder; +import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.percolate.PercolateResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.action.support.nodes.NodesOperationRequest; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.collect.UnmodifiableIterator; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.*; -import org.elasticsearch.indices.IndexMissingException; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.node.Node; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -68,8 +73,8 @@ import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.global.Global; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.missing.MissingBuilder; import org.elasticsearch.search.aggregations.bucket.range.RangeBuilder; import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeBuilder; @@ -86,10 +91,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.io.InputStreamReader; -import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; +import java.nio.file.Paths; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; @@ -99,8 +107,32 @@ import static org.elasticsearch.node.NodeBuilder.nodeBuilder; @SuppressWarnings("rawtypes") public class ElasticSearchPersistenceServiceImpl implements PersistenceService, ClusterService, SynchronousBundleListener { - public static final long MILLIS_PER_DAY = 24L * 60L * 60L * 1000L; private static final Logger logger = LoggerFactory.getLogger(ElasticSearchPersistenceServiceImpl.class.getName()); + + public static final String DISCOVERY_ZEN_PING_MULTICAST_ENABLED = "discovery.zen.ping.multicast.enabled"; + public static final String CONTEXTSERVER_ADDRESS = "contextserver.address"; + public static final String CONTEXTSERVER_PORT = "contextserver.port"; + public static final String CONTEXTSERVER_SECURE_ADDRESS = "contextserver.secureAddress"; + public static final String CONTEXTSERVER_SECURE_PORT = "contextserver.securePort"; + public static final String KARAF_HOME = "karaf.home"; + public static final String ELASTICSEARCH_HOME_DIRECTORY = "elasticsearch"; + public static final String ELASTICSEARCH_PLUGINS_DIRECTORY = ELASTICSEARCH_HOME_DIRECTORY + "/plugins"; + public static final String ELASTICSEARCH_DATA_DIRECTORY = ELASTICSEARCH_HOME_DIRECTORY + "/data"; + public static final String INDEX_NUMBER_OF_REPLICAS = "index.number_of_replicas"; + public static final String INDEX_NUMBER_OF_SHARDS = "index.number_of_shards"; + public static final String NODE_CONTEXTSERVER_ADDRESS = "node.contextserver.address"; + public static final String NODE_CONTEXTSERVER_PORT = "node.contextserver.port"; + public static final String NODE_CONTEXTSERVER_SECURE_ADDRESS = "node.contextserver.secureAddress"; + public static final String NODE_CONTEXTSERVER_SECURE_PORT = "node.contextserver.securePort"; + public static final String NUMBER_OF_SHARDS = "number_of_shards"; + public static final String NUMBER_OF_REPLICAS = "number_of_replicas"; + public static final String CLUSTER_NAME = "cluster.name"; + public static final String NODE_DATA = "node.data"; + public static final String PATH_DATA = "path.data"; + public static final String PATH_HOME = "path.home"; + public static final String PATH_PLUGINS = "path.plugins"; + public static final String INDEX_MAX_RESULT_WINDOW = "index.max_result_window"; + private Node node; private Client client; private String clusterName; @@ -220,12 +252,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, Map<String, String> settings = null; if (elasticSearchConfig != null && elasticSearchConfig.length() > 0) { try { - URL elasticSearchConfigURL = new URL(elasticSearchConfig); - Settings.Builder settingsBuilder = ImmutableSettings.builder().loadFromUrl(elasticSearchConfigURL); + URI elasticSearchConfigURI = new URI(elasticSearchConfig); + Settings.Builder settingsBuilder = Settings.builder().loadFromPath(Paths.get(elasticSearchConfigURI)); settings = settingsBuilder.build().getAsMap(); - logger.info("Successfully loaded ElasticSearch configuration from " + elasticSearchConfigURL); - } catch (MalformedURLException e) { - logger.error("Error in ElasticSearch configuration URL ", e); + logger.info("Successfully loaded ElasticSearch configuration from " + elasticSearchConfigURI); + } catch (URISyntaxException e) { + logger.error("Error in ElasticSearch configuration URI ", e); } catch (SettingsException se) { logger.info("Error trying to load settings from " + elasticSearchConfig + ": " + se.getMessage() + " (activate debug mode for exception details)"); if (logger.isDebugEnabled()) { @@ -234,25 +266,34 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, } } - address = System.getProperty("contextserver.address", address); - port = System.getProperty("contextserver.port", port); - secureAddress = System.getProperty("contextserver.secureAddress", secureAddress); - securePort = System.getProperty("contextserver.securePort", securePort); + address = System.getProperty(CONTEXTSERVER_ADDRESS, address); + port = System.getProperty(CONTEXTSERVER_PORT, port); + secureAddress = System.getProperty(CONTEXTSERVER_SECURE_ADDRESS, secureAddress); + securePort = System.getProperty(CONTEXTSERVER_SECURE_PORT, securePort); - ImmutableSettings.Builder settingsBuilder = ImmutableSettings.builder(); + Settings.Builder settingsBuilder = Settings.builder(); if (settings != null) { settingsBuilder.put(settings); } - settingsBuilder.put("cluster.name", clusterName) - .put("node.data", nodeData) - .put("discovery.zen.ping.multicast.enabled", discoveryEnabled) - .put("index.number_of_replicas", numberOfReplicas) - .put("index.number_of_shards", numberOfShards) - .put("node.contextserver.address", address) - .put("node.contextserver.port", port) - .put("node.contextserver.secureAddress", secureAddress) - .put("node.contextserver.securePort", securePort); + String karafHome = System.getProperty(KARAF_HOME); + File pluginsFile = new File(getConfig(settings, PATH_PLUGINS, new File(new File(karafHome), ELASTICSEARCH_PLUGINS_DIRECTORY).getAbsolutePath())); + File homeFile = new File(getConfig(settings, PATH_HOME, new File(new File(karafHome), ELASTICSEARCH_HOME_DIRECTORY).getAbsolutePath())); + File dataFile = new File(getConfig(settings, PATH_DATA, new File(new File(karafHome), ELASTICSEARCH_DATA_DIRECTORY).getAbsolutePath())); + + settingsBuilder.put(CLUSTER_NAME, clusterName) + .put(NODE_DATA, nodeData) + .put(PATH_DATA, dataFile.getAbsolutePath()) + .put(PATH_HOME, homeFile.getAbsolutePath()) + .put(PATH_PLUGINS, pluginsFile.getAbsolutePath()) + .put(DISCOVERY_ZEN_PING_MULTICAST_ENABLED, discoveryEnabled) + .put(INDEX_NUMBER_OF_REPLICAS, numberOfReplicas) + .put(INDEX_NUMBER_OF_SHARDS, numberOfShards) + .put(NODE_CONTEXTSERVER_ADDRESS, address) + .put(NODE_CONTEXTSERVER_PORT, port) + .put(NODE_CONTEXTSERVER_SECURE_ADDRESS, secureAddress) + .put(NODE_CONTEXTSERVER_SECURE_PORT, securePort) + .put(INDEX_MAX_RESULT_WINDOW, "2147483647"); node = nodeBuilder().settings(settingsBuilder).node(); client = node.client(); @@ -284,9 +325,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, client.admin().indices().preparePutTemplate(indexName + "_monthlyindex") .setTemplate(indexName + "-*") .setOrder(1) - .setSettings(ImmutableSettings.settingsBuilder() - .put("number_of_shards", Integer.parseInt(monthlyIndexNumberOfShards)) - .put("number_of_replicas", Integer.parseInt(monthlyIndexNumberOfReplicas)) + .setSettings(Settings.settingsBuilder() + .put(NUMBER_OF_SHARDS, Integer.parseInt(monthlyIndexNumberOfShards)) + .put(NUMBER_OF_REPLICAS, Integer.parseInt(monthlyIndexNumberOfReplicas)) .build()).execute().actionGet(); getMonthlyIndex(new Date(), true); @@ -442,7 +483,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, @Override public long getAllItemsCount(String itemType) { - return queryCount(FilterBuilders.matchAllFilter(), itemType); + return queryCount(QueryBuilders.matchAllQuery(), itemType); } @Override @@ -477,19 +518,19 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, .actionGet(); if (response.isExists()) { String sourceAsString = response.getSourceAsString(); - final T value = CustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz); + final T value = CustomObjectMapper.getObjectMapper().readValue(FieldDotEscaper.unescapeJson(sourceAsString), clazz); value.setItemId(response.getId()); return value; } else { return null; } } - } catch (IndexMissingException e) { - logger.debug("No index found for itemType=" + clazz.getName() + "itemId=" + itemId, e); + } catch (IndexNotFoundException e) { + logger.debug("No index found for itemType=" + clazz.getName() + " itemId=" + itemId, e); } catch (IllegalAccessException e) { - logger.error("Error loading itemType=" + clazz.getName() + "itemId=" + itemId, e); + logger.error("Error loading itemType=" + clazz.getName() + " itemId=" + itemId, e); } catch (Exception t) { - logger.error("Error loading itemType=" + clazz.getName() + "itemId=" + itemId, t); + logger.error("Error loading itemType=" + clazz.getName() + " itemId=" + itemId, t); } return null; } @@ -504,6 +545,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, protected Boolean execute(Object... args) { try { String source = CustomObjectMapper.getObjectMapper().writeValueAsString(item); + Set<String> modifiedNames = new LinkedHashSet<>(); + source = FieldDotEscaper.escapeJson(source, modifiedNames); + if (modifiedNames.size() > 0) { + logger.warn("Found JSON property names with dot characters not allowed by ElasticSearch 2.x={} in item {}", modifiedNames, item); + } String itemType = item.getItemType(); String index = indexNames.containsKey(itemType) ? indexNames.get(itemType) : (itemsMonthlyIndexed.contains(itemType) ? getMonthlyIndex(((TimestampedItem) item).getTimeStamp()) : indexName); @@ -514,7 +560,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, } try { indexBuilder.execute().actionGet(); - } catch (IndexMissingException e) { + } catch (IndexNotFoundException e) { if (itemsMonthlyIndexed.contains(itemType)) { getMonthlyIndex(((TimestampedItem) item).getTimeStamp(), true); indexBuilder.execute().actionGet(); @@ -545,11 +591,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, String index = indexNames.containsKey(itemType) ? indexNames.get(itemType) : (itemsMonthlyIndexed.contains(itemType) && dateHint != null ? getMonthlyIndex(dateHint) : indexName); - client.prepareUpdate(index, itemType, itemId).setDoc(source) + client.prepareUpdate(index, itemType, itemId).setDoc(FieldDotEscaper.escapeMap(source)) .execute() .actionGet(); return true; - } catch (IndexMissingException e) { + } catch (IndexNotFoundException e) { logger.debug("No index found for itemType=" + clazz.getName() + "itemId=" + itemId, e); } catch (NoSuchFieldException e) { logger.error("Error updating item " + itemId, e); @@ -571,11 +617,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, String index = indexNames.containsKey(itemType) ? indexNames.get(itemType) : (itemsMonthlyIndexed.contains(itemType) && dateHint != null ? getMonthlyIndex(dateHint) : indexName); - client.prepareUpdate(index, itemType, itemId).setScript(script, ScriptService.ScriptType.INLINE).setScriptParams(scriptParams) + Script actualScript = new Script(script, ScriptService.ScriptType.INLINE, null, scriptParams); + client.prepareUpdate(index, itemType, itemId).setScript(actualScript) .execute() .actionGet(); return true; - } catch (IndexMissingException e) { + } catch (IndexNotFoundException e) { logger.debug("No index found for itemType=" + clazz.getName() + "itemId=" + itemId, e); } catch (NoSuchFieldException e) { logger.error("Error updating item " + itemId, e); @@ -612,9 +659,12 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, try { String itemType = (String) clazz.getField("ITEM_TYPE").get(null); - client.prepareDeleteByQuery(getIndexNameForQuery(itemType)) + DeleteByQueryResponse rsp = new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) + .setIndices(getIndexNameForQuery(itemType)) .setQuery(conditionESQueryBuilderDispatcher.getQueryBuilder(query)) - .execute().actionGet(); + .execute() + .actionGet(); + return true; } catch (Exception e) { logger.error("Cannot remove by query", e); @@ -817,8 +867,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, final Class<? extends Item> clazz = item.getClass(); String itemType = (String) clazz.getField("ITEM_TYPE").get(null); - FilterBuilder builder = FilterBuilders.andFilter( - FilterBuilders.idsFilter(itemType).ids(item.getItemId()), + QueryBuilder builder = QueryBuilders.andQuery( + QueryBuilders.idsQuery(itemType).ids(item.getItemId()), conditionESQueryBuilderDispatcher.buildFilter(query)); return queryCount(builder, itemType) > 0; } catch (IllegalAccessException e) { @@ -882,7 +932,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, return queryCount(conditionESQueryBuilderDispatcher.buildFilter(query), itemType); } - private long queryCount(final FilterBuilder filter, final String itemType) { + private long queryCount(final QueryBuilder filter, final String itemType) { return new InClassLoaderExecute<Long>() { @Override @@ -960,7 +1010,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, results.add(value); } } catch (Exception t) { - logger.error("Error loading itemType=" + clazz.getName() + "query=" + query, t); + logger.error("Error loading itemType=" + clazz.getName() + " query=" + query + " sortBy=" + sortBy, t); } return new PartialList<T>(results, offset, size, totalHits); @@ -987,7 +1037,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, AggregationBuilder bucketsAggregation = null; if (aggregate instanceof DateAggregate) { DateAggregate dateAggregate = (DateAggregate) aggregate; - DateHistogramBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram("buckets").field(aggregate.getField()).interval(new DateHistogram.Interval((dateAggregate.getInterval()))); + DateHistogramBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram("buckets").field(aggregate.getField()).interval(new DateHistogramInterval((dateAggregate.getInterval()))); if (dateAggregate.getFormat() != null) { dateHistogramBuilder.format(dateAggregate.getFormat()); } @@ -1073,7 +1123,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, if (aggregations.get("buckets") != null) { MultiBucketsAggregation terms = aggregations.get("buckets"); for (MultiBucketsAggregation.Bucket bucket : terms.getBuckets()) { - results.put(bucket.getKey(), bucket.getDocCount()); + results.put(bucket.getKeyAsString(), bucket.getDocCount()); } SingleBucketAggregation missing = aggregations.get("missing"); if (missing.getDocCount() > 0) { @@ -1116,7 +1166,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, protected List<ClusterNode> execute(Object... args) { Map<String, ClusterNode> clusterNodes = new LinkedHashMap<String, ClusterNode>(); - NodesInfoResponse nodesInfoResponse = client.admin().cluster().prepareNodesInfo(NodesOperationRequest.ALL_NODES) + NodesInfoResponse nodesInfoResponse = client.admin().cluster().prepareNodesInfo(NodesInfoRequest.ALL_NODES) .setSettings(true) .execute() .actionGet(); @@ -1135,8 +1185,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, } } - NodesStatsResponse nodesStatsResponse = client.admin().cluster().prepareNodesStats(NodesOperationRequest.ALL_NODES) + NodesStatsResponse nodesStatsResponse = client.admin().cluster().prepareNodesStats(NodesInfoRequest.ALL_NODES) .setOs(true) + .setJvm(true) .setProcess(true) .execute() .actionGet(); @@ -1152,8 +1203,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, clusterNode.setCpuLoad(nodeStats.getProcess().getCpu().getPercent()); } if (nodeStats.getOs() != null) { - clusterNode.setLoadAverage(nodeStats.getOs().getLoadAverage()); - clusterNode.setUptime(nodeStats.getOs().getUptime().getMillis()); + clusterNode.setLoadAverage(new double[] { nodeStats.getOs().getLoadAverage() }); + clusterNode.setUptime(nodeStats.getJvm().getUptime().getMillis()); } } } @@ -1335,5 +1386,11 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, } } - + private String getConfig(Map<String,String> settings, String key, + String defaultValue) { + if (settings != null && settings.get(key) != null) { + return settings.get(key); + } + return defaultValue; + } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java new file mode 100644 index 0000000..62ccacc --- /dev/null +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotEscaper.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.persistence.elasticsearch; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +/** + * A utility class to un/escape field names that contain dots that are not allowed in ElasticSearch 2.x + */ +public class FieldDotEscaper { + + public static final String DOT_ESCAPE_MARKER = "__DOT__"; + + public static String escapeJson(final String jsonInput) { + return escapeJson(jsonInput, null); + } + + public static String escapeJson(final String jsonInput, final Set<String> modifiedNames) { + if (!jsonInput.contains(".")) { // optimization in case no dot is present at all + return jsonInput; + } + StringBuffer result = new StringBuffer(); + FieldDotJsonTransformer jsonTransformer = new FieldDotJsonTransformer(jsonInput, result, DOT_ESCAPE_MARKER); + Set<String> pathsModified = jsonTransformer.transform(); + if (modifiedNames != null) { + modifiedNames.addAll(pathsModified); + } + return result.toString(); + } + + public static String unescapeJson(final String jsonInput) { + return unescapeString(jsonInput); + } + + public static String escapeString(final String stringInput) { + return stringInput.replaceAll("\\.", DOT_ESCAPE_MARKER); + } + + public static String unescapeString(final String stringInput) { + return stringInput.replaceAll(DOT_ESCAPE_MARKER, "."); + } + + public static Map<? extends String, ?> escapeMap(final Map<? extends String, ?> mapInput) { + Map<String,Object> result = new LinkedHashMap<>(mapInput.size()); + for (Map.Entry<? extends String, ? extends Object> entry : mapInput.entrySet()) { + String entryKey = entry.getKey(); + if (entryKey.contains(".")) { + entryKey = escapeString(entryKey); + } + result.put(entryKey, entry.getValue()); + } + return result; + } + + public static Map<? extends String, ?> unescapeMap(final Map<? extends String, ?> mapInput) { + Map<String, Object> result = new LinkedHashMap<>(mapInput.size()); + for (Map.Entry<? extends String, ?> entry : mapInput.entrySet()) { + String entryKey = entry.getKey(); + if (entryKey.contains(DOT_ESCAPE_MARKER)) { + entryKey = unescapeString(entryKey); + } + result.put(entryKey, entry.getValue()); + } + return result; + } + + public static Properties escapeProperties(final Properties input) { + Properties result = new Properties(); + for (String propertyName : input.stringPropertyNames()) { + String newPropertyName = propertyName; + if (propertyName.contains(".")) { + newPropertyName = escapeString(propertyName); + } + result.put(newPropertyName, input.getProperty(propertyName)); + } + return result; + } + + public static Properties unescapeProperties(final Properties input) { + Properties result = new Properties(); + for (String propertyName : input.stringPropertyNames()) { + String newPropertyName = propertyName; + if (propertyName.contains(DOT_ESCAPE_MARKER)) { + newPropertyName = unescapeString(propertyName); + } + result.put(newPropertyName, input.getProperty(propertyName)); + } + return result; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java new file mode 100644 index 0000000..a26e3f4 --- /dev/null +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/FieldDotJsonTransformer.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.persistence.elasticsearch; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Deque; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.Set; + +/** + * A custom JSON transformer that can replace dot characters in field names with a marker. This is useful for tools like + * ElasticSearch 2.x that doesn't allow dot characters in field names since version 2.x + */ +public class FieldDotJsonTransformer { + + public static final char BEGIN_ARRAY_CHAR = '['; + public static final char END_ARRAY_CHAR = ']'; + public static final char BEGIN_OBJECT_CHAR = '{'; + public static final char END_OBJECT_CHAR = '}'; + public static final char NAME_SEPARATOR_CHAR = ':'; + public static final char VALUE_SEPARATOR_CHAR = ','; + + public static final char STRING_BEGIN_OR_END_CHAR = '"'; + public static final char STRING_ESCAPE_CHAR = '\\'; + public static final char STRING_UNICODE_CHAR = 'u'; + public static final char STRING_DOT_CHAR = '.'; + + public static final String WHITESPACE_CHARS = " \t\n\r"; + public static final String NUMBER_CHARS = "+-0123456789eE."; + + String jsonInput; + int pos = -1; + StringBuffer output; + String dotReplacement = null; + Set<String> modifiedNames = new LinkedHashSet<>(); + Deque<String> currentPath = new LinkedList<>(); + + public FieldDotJsonTransformer(String jsonInput, StringBuffer output, String dotReplacement) { + this.jsonInput = jsonInput; + this.output = output; + this.dotReplacement = dotReplacement; + } + + public Set<String> transform() { + parseValue(); + return modifiedNames; + } + + protected Character getNextChar() { + pos++; + char ch = jsonInput.charAt(pos); + if (pos >= jsonInput.length()) { + return null; + } + return ch; + } + + protected Character peekNextToken() { + if (pos + 1 >= jsonInput.length()) { + return null; + } + int i = 1; + Character ch = jsonInput.charAt(pos + i); + while (WHITESPACE_CHARS.indexOf(ch) > -1 && (pos + i < jsonInput.length())) { + i++; + ch = jsonInput.charAt(pos + i); + } + return ch; + } + + protected Character getNextToken() { + Character ch = getNextChar(); + while ((ch != null) && (WHITESPACE_CHARS.indexOf(ch) > -1)) { + output.append(ch); + ch = getNextChar(); + } + return ch; + } + + protected void parseBooleanValue(boolean expectedValue) { + if (expectedValue) { + StringBuilder sb = new StringBuilder(); + sb.append(getNextToken()); + sb.append(getNextChar()); + sb.append(getNextChar()); + sb.append(getNextChar()); + if ("true".equals(sb.toString())) { + // everything matches + } + output.append(sb.toString()); + } else { + StringBuilder sb = new StringBuilder(); + sb.append(getNextToken()); + sb.append(getNextChar()); + sb.append(getNextChar()); + sb.append(getNextChar()); + sb.append(getNextChar()); + if ("false".equals(sb.toString())) { + // everything matches + } + output.append(sb.toString()); + } + } + + protected void parseNullValue() { + StringBuilder sb = new StringBuilder(); + sb.append(getNextToken()); + sb.append(getNextChar()); + sb.append(getNextChar()); + sb.append(getNextChar()); + if ("null".equals(sb.toString())) { + // everything matches + } + output.append(sb.toString()); + } + + protected String parseString(boolean escapeDots) { + Character ch = getNextToken(); + if (ch != STRING_BEGIN_OR_END_CHAR) { + return null; + } + output.append(ch); + boolean modified = false; + StringBuilder stringContent = new StringBuilder(); + while ((ch = getNextChar()) != null) { + switch (ch) { + case STRING_ESCAPE_CHAR: + stringContent.append(ch); + output.append(ch); + ch = getNextChar(); + if (ch == STRING_UNICODE_CHAR) { + // case of Unicode escape sequence + } + output.append(ch); + stringContent.append(ch); + break; + case STRING_DOT_CHAR: + if (escapeDots && dotReplacement != null) { + output.append(dotReplacement); + modified = true; + } else { + output.append(ch); + } + stringContent.append(ch); + break; + case STRING_BEGIN_OR_END_CHAR: + output.append(ch); + if (modified) { + modifiedNames.add(StringUtils.join(currentPath, "/") + "/" + stringContent.toString()); + } + return stringContent.toString(); + default: + output.append(ch); + stringContent.append(ch); + } + } + return null; + } + + protected void parseNumber() { + StringBuilder sb = new StringBuilder(); + Character ch = peekNextToken(); + while ((ch != null) && (NUMBER_CHARS.indexOf(ch) > -1)) { + ch = getNextChar(); + sb.append(ch); + ch = peekNextToken(); + } + output.append(sb.toString()); + } + + protected void parseValue() { + char ch = peekNextToken(); + // we've got to identify the type or value first + switch (ch) { + case 't': // true + parseBooleanValue(true); + break; + case 'f': // false + parseBooleanValue(false); + break; + case 'n': // null + parseNullValue(); + break; + case BEGIN_OBJECT_CHAR: + parseObject(); + break; + case BEGIN_ARRAY_CHAR: + parseArray(); + break; + case STRING_BEGIN_OR_END_CHAR: + parseString(false); + break; + default: + parseNumber(); + } + } + + protected void parseObject() { + Character ch = getNextToken(); + if (ch != BEGIN_OBJECT_CHAR) { + return; + } + output.append(ch); + // now let's check the case of an empty object + ch = peekNextToken(); + if (ch == END_OBJECT_CHAR) { + ch = getNextToken(); + output.append(ch); + return; + } + if (parseNameValuePair()) { + return; + } + while ((ch = getNextToken()) != null) { + output.append(ch); + switch (ch) { + case VALUE_SEPARATOR_CHAR: + parseNameValuePair(); + break; + case END_OBJECT_CHAR: + return; + default: + return; + } + } + } + + protected void parseArray() { + Character ch = getNextToken(); + if (ch != BEGIN_ARRAY_CHAR) { + return; + } + output.append(ch); + // now let's check the case of an empty array + ch = peekNextToken(); + if (ch == END_ARRAY_CHAR) { + ch = getNextToken(); + output.append(ch); + return; + } + parseValue(); + while ((ch = getNextToken()) != null) { + output.append(ch); + switch (ch) { + case VALUE_SEPARATOR_CHAR: + parseValue(); + break; + case END_ARRAY_CHAR: + return; + default: + return; + } + } + } + + protected boolean parseNameValuePair() { + Character ch; + String name = parseString(true); + if (name != null) { + currentPath.addLast(name); + } + ch = getNextToken(); + if (ch != NAME_SEPARATOR_CHAR) { + return true; + } + output.append(ch); + parseValue(); + if (name != null) { + currentPath.removeLast(); + } + return false; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java index 3666d8d..63964e1 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java @@ -17,12 +17,12 @@ package org.apache.unomi.persistence.elasticsearch.conditions; +import com.google.common.base.Function; +import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; import org.apache.lucene.util.ArrayUtil; import org.apache.unomi.api.conditions.Condition; -import org.elasticsearch.common.base.Function; -import org.elasticsearch.common.collect.Lists; import org.mvel2.MVEL; import org.mvel2.ParserConfiguration; import org.mvel2.ParserContext; http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java index cc2dc89..78a5cba 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java @@ -18,12 +18,12 @@ package org.apache.unomi.persistence.elasticsearch.conditions; import org.apache.unomi.api.conditions.Condition; -import org.elasticsearch.index.query.FilterBuilder; +import org.elasticsearch.index.query.QueryBuilder; import java.util.Map; public interface ConditionESQueryBuilder { - FilterBuilder buildFilter(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher); + QueryBuilder buildQuery(Condition condition, Map<String, Object> context, ConditionESQueryBuilderDispatcher dispatcher); } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java index 526b49f..5289f69 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java @@ -18,9 +18,7 @@ package org.apache.unomi.persistence.elasticsearch.conditions; import org.apache.unomi.api.conditions.Condition; -import org.elasticsearch.index.query.FilterBuilder; -import org.elasticsearch.index.query.FilterBuilders; -import org.elasticsearch.index.query.FilteredQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.osgi.framework.BundleContext; import org.slf4j.Logger; @@ -67,15 +65,15 @@ public class ConditionESQueryBuilderDispatcher { return "{\"query\": " + getQueryBuilder(condition).toString() + "}"; } - public FilteredQueryBuilder getQueryBuilder(Condition condition) { - return QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), buildFilter(condition)); + public QueryBuilder getQueryBuilder(Condition condition) { + return QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()).filter(buildFilter(condition)); } - public FilterBuilder buildFilter(Condition condition) { + public QueryBuilder buildFilter(Condition condition) { return buildFilter(condition, new HashMap<String, Object>()); } - public FilterBuilder buildFilter(Condition condition, Map<String, Object> context) { + public QueryBuilder buildFilter(Condition condition, Map<String, Object> context) { if(condition == null || condition.getConditionType() == null) { throw new IllegalArgumentException("Condition is null or doesn't have type, impossible to build filter"); } @@ -94,12 +92,12 @@ public class ConditionESQueryBuilderDispatcher { ConditionESQueryBuilder queryBuilder = queryBuilders.get(queryBuilderKey); Condition contextualCondition = ConditionContextHelper.getContextualCondition(condition, context); if (contextualCondition != null) { - return queryBuilder.buildFilter(contextualCondition, context, this); + return queryBuilder.buildQuery(contextualCondition, context, this); } } // if no matching - return FilterBuilders.matchAllFilter(); + return QueryBuilders.matchAllQuery(); } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json index c469be8..36788d7 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json @@ -22,11 +22,11 @@ }, "startDate": { "type": "date", - "format": "dateOptionalTime" + "format": "strict_date_optional_time||epoch_millis" }, "endDate": { "type": "date", - "format": "dateOptionalTime" + "format": "strict_date_optional_time||epoch_millis" }, "itemId": { "type": "string", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json index e9ce4d3..ccc001e 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json @@ -26,7 +26,7 @@ }, "eventDate": { "type": "date", - "format": "dateOptionalTime" + "format": "strict_date_optional_time||epoch_millis" }, "itemId": { "type": "string", http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json index abe1dbc..6089499 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json @@ -39,7 +39,7 @@ }, "timeStamp": { "type": "date", - "format": "dateOptionalTime" + "format": "strict_date_optional_time||epoch_millis" } } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json index bfd6939..4a20426 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/profile.json @@ -31,7 +31,7 @@ "properties": { "firstVisit": { "type": "date", - "format": "dateOptionalTime" + "format": "strict_date_optional_time||epoch_millis" } } } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/131a06aa/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json index ef808ed..2047b4e 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/session.json @@ -26,7 +26,7 @@ }, "lastEventDate": { "type": "date", - "format": "dateOptionalTime" + "format": "strict_date_optional_time||epoch_millis" }, "profileId": { "type": "string", @@ -51,7 +51,7 @@ }, "timeStamp": { "type": "date", - "format": "dateOptionalTime" + "format": "strict_date_optional_time||epoch_millis" } } }
