Repository: incubator-unomi Updated Branches: refs/heads/master 9742f7bb8 -> 97df6d1cd
UNOMI-104 Upgrade ElasticSearch support to 5.6 - Upgrade ES support to 5.6 - Update documentation - Add new document on how to migrate ES Signed-off-by: Serge Huber <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/97df6d1c Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/97df6d1c Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/97df6d1c Branch: refs/heads/master Commit: 97df6d1cd263161effa4f1b0e4dd755508ce2245 Parents: 9742f7b Author: Serge Huber <[email protected]> Authored: Thu Oct 12 14:17:35 2017 +0200 Committer: Serge Huber <[email protected]> Committed: Thu Oct 12 14:17:35 2017 +0200 ---------------------------------------------------------------------- .../java/org/apache/unomi/itests/BaseIT.java | 3 +- persistence-elasticsearch/core/pom.xml | 2 + .../ElasticSearchPersistenceServiceImpl.java | 19 ++------ .../resources/OSGI-INF/blueprint/blueprint.xml | 4 +- ...g.apache.unomi.persistence.elasticsearch.cfg | 3 +- ...ocationByPointSessionConditionEvaluator.java | 2 +- pom.xml | 2 +- src/site/markdown/index.md | 13 +++-- src/site/markdown/main.md | 4 ++ src/site/markdown/migrating-elasticsearch.md | 50 ++++++++++++++++++++ .../versions/master/building-and-deploying.md | 24 ++++++---- 11 files changed, 91 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/97df6d1c/itests/src/test/java/org/apache/unomi/itests/BaseIT.java ---------------------------------------------------------------------- diff --git a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java index ec0bc72..3f88d01 100644 --- a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java @@ -19,7 +19,6 @@ package org.apache.unomi.itests; import org.ops4j.pax.exam.Configuration; import org.ops4j.pax.exam.Option; -import org.ops4j.pax.exam.karaf.options.KarafDistributionConfigurationFileReplacementOption; import org.ops4j.pax.exam.karaf.options.LogLevelOption.LogLevel; import org.ops4j.pax.exam.options.MavenArtifactUrlReference; import org.ops4j.pax.exam.options.MavenUrlReference; @@ -115,6 +114,8 @@ public abstract class BaseIT { systemProperty("org.ops4j.pax.exam.rbc.rmi.port").value("1199"), systemProperty("org.apache.unomi.itests.elasticsearch.transport.port").value("9500"), systemProperty("org.apache.unomi.itests.elasticsearch.http.port").value("9400"), + systemProperty("unomi.autoStart").value("true"), + features(karafPaxWebRepo, "shell"), features(karafPaxWebRepo, "war"), features(karafCxfRepo, "cxf"), features(karafCellarRepo, "cellar"), http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/97df6d1c/persistence-elasticsearch/core/pom.xml ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml index 5254542..198392f 100644 --- a/persistence-elasticsearch/core/pom.xml +++ b/persistence-elasticsearch/core/pom.xml @@ -215,6 +215,8 @@ com.fasterxml.jackson.dataformat.xml;resolution:=optional, com.fasterxml.jackson.dataformat.xml.annotation;resolution:=optional, com.fasterxml.jackson.dataformat.xml.util;resolution:=optional, + io.netty.internal.tcnative;resolution:=optional, + org.conscrypt;resolution:=optional, * </Import-Package> <Export-Package> http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/97df6d1c/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java index f6d570e..4b2df2e 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java @@ -55,7 +55,7 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.index.reindex.BulkIndexByScrollResponse; +import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.UpdateByQueryAction; import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder; import org.elasticsearch.script.Script; @@ -134,7 +134,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, private Timer timer; - private String bulkProcessorName = "unomi-bulk"; private String bulkProcessorConcurrentRequests = "1"; private String bulkProcessorBulkActions = "1000"; private String bulkProcessorBulkSize = "5MB"; @@ -142,7 +141,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, private String bulkProcessorBackoffPolicy = "exponential"; private String minimalElasticSearchVersion = "5.0.0"; - private String maximalElasticSearchVersion = "5.3.0"; + private String maximalElasticSearchVersion = "5.7.0"; private String aggregateQueryBucketSize = "5000"; @@ -209,10 +208,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, this.conditionESQueryBuilderDispatcher = conditionESQueryBuilderDispatcher; } - public void setBulkProcessorName(String bulkProcessorName) { - this.bulkProcessorName = bulkProcessorName; - } - public void setBulkProcessorConcurrentRequests(String bulkProcessorConcurrentRequests) { this.bulkProcessorConcurrentRequests = bulkProcessorConcurrentRequests; } @@ -254,7 +249,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, public Object execute(Object... args) throws Exception { logger.info("Connecting to ElasticSearch persistence backend using cluster name " + clusterName + " and index name " + indexName + "..."); - bulkProcessorName = System.getProperty(BULK_PROCESSOR_NAME, bulkProcessorName); bulkProcessorConcurrentRequests = System.getProperty(BULK_PROCESSOR_CONCURRENT_REQUESTS, bulkProcessorConcurrentRequests); bulkProcessorBulkActions = System.getProperty(BULK_PROCESSOR_BULK_ACTIONS, bulkProcessorBulkActions); bulkProcessorBulkSize = System.getProperty(BULK_PROCESSOR_BULK_SIZE, bulkProcessorBulkSize); @@ -437,9 +431,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, logger.error("After Bulk (failure)", failure); } }); - if (bulkProcessorName != null && bulkProcessorName.length() > 0) { - bulkProcessorBuilder.setName(bulkProcessorName); - } if (bulkProcessorConcurrentRequests != null) { int concurrentRequests = Integer.parseInt(bulkProcessorConcurrentRequests); if (concurrentRequests > 1) { @@ -787,7 +778,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, UpdateByQueryRequestBuilder ubqrb = UpdateByQueryAction.INSTANCE.newRequestBuilder(client); ubqrb.source(index).source().setTypes(itemType); - BulkIndexByScrollResponse response = ubqrb.setSlices(2) + BulkByScrollResponse response = ubqrb.setSlices(2) .setMaxRetries(1000).abortOnVersionConflict(false).script(actualScript) .filter(conditionESQueryBuilderDispatcher.buildFilter(conditions[i])).get(); if (response.getBulkFailures().size() > 0) { @@ -1067,8 +1058,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, } } } - } catch (IOException e) { - throw new Exception("Cannot get mapping", e); + } catch (Throwable t) { + throw new Exception("Cannot get mapping for itemType="+ itemType, t); } return propertyMap; } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/97df6d1c/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 0ca1857..ad095c1 100644 --- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -39,7 +39,6 @@ <cm:property name="monthlyIndex.numberOfReplicas" value="0"/> <cm:property name="defaultQueryLimit" value="10"/> - <cm:property name="bulkProcessor.name" value="unomi-bulk" /> <cm:property name="bulkProcessor.concurrentRequests" value="1" /> <cm:property name="bulkProcessor.bulkActions" value="1000" /> <cm:property name="bulkProcessor.bulkSize" value="5MB" /> @@ -47,7 +46,7 @@ <cm:property name="bulkProcessor.backoffPolicy" value="exponential" /> <cm:property name="minimalElasticSearchVersion" value="5.0.0" /> - <cm:property name="maximalElasticSearchVersion" value="5.3.0" /> + <cm:property name="maximalElasticSearchVersion" value="5.7.0" /> <cm:property name="aggregateQueryBucketSize" value="5000" /> @@ -98,7 +97,6 @@ <entry key="geonameEntry" value="geonames"/> </map> </property> - <property name="bulkProcessorName" value="${es.bulkProcessor.name}" /> <property name="bulkProcessorConcurrentRequests" value="${es.bulkProcessor.concurrentRequests}" /> <property name="bulkProcessorBulkActions" value="${es.bulkProcessor.bulkActions}" /> <property name="bulkProcessorBulkSize" value="${es.bulkProcessor.bulkSize}" /> http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/97df6d1c/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg index a77b34e..cd9d14d 100644 --- a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg +++ b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg @@ -30,7 +30,6 @@ defaultQueryLimit=10 # The following settings control the behavior of the BulkProcessor API. You can find more information about these # settings and their behavior here : https://www.elastic.co/guide/en/elasticsearch/client/java-api/2.4/java-docs-bulk-processor.html # The values used here are the default values of the API -bulkProcessor.name=unomi-bulk bulkProcessor.concurrentRequests=1 bulkProcessor.bulkActions=1000 bulkProcessor.bulkSize=5MB @@ -42,7 +41,7 @@ bulkProcessor.backoffPolicy=exponential # for each node in the ElasticSearch cluster: # minimalElasticSearchVersion <= ElasticSearch node version < maximalElasticSearchVersion minimalElasticSearchVersion=5.0.0 -maximalElasticSearchVersion=5.3.0 +maximalElasticSearchVersion=5.7.0 # The following setting is used to set the aggregate query bucket size aggregateQueryBucketSize=5000 http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/97df6d1c/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java ---------------------------------------------------------------------- diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java index c0b4134..8de237b 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java @@ -45,7 +45,7 @@ public class GeoLocationByPointSessionConditionEvaluator implements ConditionEva Double circleLongitude = (Double) condition.getParameter("circleLongitude"); DistanceUnit.Distance distance = DistanceUnit.Distance.parseDistance(condition.getParameter("distance").toString()); - double d = GeoDistance.DEFAULT.calculate(circleLatitude, circleLongitude, latitudeProperty, longitudeProperty, distance.unit); + double d = GeoDistance.PLANE.calculate(circleLatitude, circleLongitude, latitudeProperty, longitudeProperty, distance.unit); return d < distance.value; } else if("rectangle".equals(type)) { Double rectLatitudeNE = (Double) condition.getParameter("rectLatitudeNE"); http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/97df6d1c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index e1e9b38..88a6933 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,7 @@ <version.karaf>3.0.8</version.karaf> <version.karaf.cellar>3.0.3</version.karaf.cellar> <version.pax.exam>4.11.0</version.pax.exam> - <elasticsearch.version>5.2.2</elasticsearch.version> + <elasticsearch.version>5.6.3</elasticsearch.version> <maven.compiler.source>1.7</maven.compiler.source> <maven.compiler.target>1.7</maven.compiler.target> http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/97df6d1c/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/src/site/markdown/index.md b/src/site/markdown/index.md index c816be5..5e7ab5e 100644 --- a/src/site/markdown/index.md +++ b/src/site/markdown/index.md @@ -73,11 +73,14 @@ All communication is done using HTTP REST requests and JSON data formats. ## 5 Minute Quick start 1. Install JDK 8 (see [http://www.oracle.com/technetwork/java/javase/downloads/index.html](http://www.oracle.com/technetwork/java/javase/downloads/index.html) and make sure you set the JAVA_HOME variable [https://docs.oracle.com/cd/E19182-01/820-7851/inst_cli_jdk_javahome_t/](https://docs.oracle.com/cd/E19182-01/820-7851/inst_cli_jdk_javahome_t/) -2. Download Apache Unomi here : [http://unomi.incubator.apache.org/download.html](http://unomi.incubator.apache.org/download.html) -3. Start it using : `./bin/karaf` -4. Wait for startup to complete -5. Try accessing [https://localhost:9443/cxs/cluster](https://localhost:9443/cxs/cluster) with username/password: karaf/karaf . You might get a certificate warning in your browser, just accept it despite the warning it is safe. -6. Request your first context by simply accessing : [http://localhost:8181/context.js?sessionId=1234](http://localhost:8181/context.js?sessionId=1234) +2. Download ElasticSearch here : [https://www.elastic.co/downloads/elasticsearch](https://www.elastic.co/downloads/elasticsearch) (use version 5.1.2 for Unomi <= 1.2, 5.6.3 for Unomi >= 1.3) +3. Uncompress it and change the `config/elasticsearch.yml` to include the following config : `cluster.name: contextElasticSearch` +4. Launch ElasticSearch using : `bin/elasticsearch` +5. Download Apache Unomi here : [http://unomi.incubator.apache.org/download.html](http://unomi.incubator.apache.org/download.html) +6. Start it using : `./bin/karaf` +7. Wait for startup to complete +8. Try accessing [https://localhost:9443/cxs/cluster](https://localhost:9443/cxs/cluster) with username/password: karaf/karaf . You might get a certificate warning in your browser, just accept it despite the warning it is safe. +9. Request your first context by simply accessing : [http://localhost:8181/context.js?sessionId=1234](http://localhost:8181/context.js?sessionId=1234) ### News http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/97df6d1c/src/site/markdown/main.md ---------------------------------------------------------------------- diff --git a/src/site/markdown/main.md b/src/site/markdown/main.md index 7c79ac5..00cd81f 100644 --- a/src/site/markdown/main.md +++ b/src/site/markdown/main.md @@ -48,3 +48,7 @@ In this section we regroup the documentation by Apache Unomi version. - [Cluster setup](versions/master/clustering.html) - [Concepts](versions/master/concepts.html) - [Custom extensions](versions/master/custom-extensions.html) + +# Migration documentation + +- [ElasticSearch migrations](migrating-elasticsearch.html) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/97df6d1c/src/site/markdown/migrating-elasticsearch.md ---------------------------------------------------------------------- diff --git a/src/site/markdown/migrating-elasticsearch.md b/src/site/markdown/migrating-elasticsearch.md new file mode 100644 index 0000000..6b5d4fd --- /dev/null +++ b/src/site/markdown/migrating-elasticsearch.md @@ -0,0 +1,50 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +# Migrating ElasticSearch + +## Introduction + +In this section of the documentation we will explain some general notions of how to migrate between ElasticSearch +versions, as well as present specific migration steps to migrate from one version of ElasticSearch used by Apache Unomi +to another, if it is needed. + +## General steps and notions + +Depending on the type of ElasticSearch install you may have, the migration steps will differ. Basically when dealing with +a single node (standalone) installation, a simple procedure may be used that simply copies some directories over, while +in the case of a cluster installation ElasticSearch Snapshot and Restore functionality must be used. + +### Standalone (one node migration) + +In the case of a standalone install, it is generally sufficient, provided the versions are compatible (meaning that only +one major version seperates the two installs), to simply copy over the `data` directory over to the new version. Also +you will need to make sure that you copy over any custom settings from the `config/elasticsearch.yml` file over to the +new version. + +### Cluster migration + +Here we recommend you read the [official upgrading documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html). + +## Migrating between versions used by Apache Unomi. + +### ElasticSearch 5.1.2 to 5.6.3 (Unomi 1.2.0 -> 1.3.0) + +Steps: + +1. Depending on your install, perform either the standalone or cluster migration +2. That's it ! \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/97df6d1c/src/site/markdown/versions/master/building-and-deploying.md ---------------------------------------------------------------------- diff --git a/src/site/markdown/versions/master/building-and-deploying.md b/src/site/markdown/versions/master/building-and-deploying.md index f4fbfb8..56c2cc6 100644 --- a/src/site/markdown/versions/master/building-and-deploying.md +++ b/src/site/markdown/versions/master/building-and-deploying.md @@ -56,18 +56,26 @@ Installing an ElasticSearch server Starting with version 1.2, Apache Unomi no longer embeds an ElasticSearch server as this is no longer supported by the developers of ElasticSearch. Therefore you will need to install a standalone ElasticSearch using the following steps: -1. Download an ElasticSearch 5.x version (5.1.1 or more recent, but not 6.x) from the following site: +1. Download an ElasticSearch 5.x version (5.1.1 or more recent, but not 6.x). Here's the version you will need depending +on your version of Apache Unomi. - https://www.elastic.co/downloads/elasticsearch - -2. Uncompress the downloaded package into a directory and launch the server using + Apache Unomi <= 1.2 : [https://www.elastic.co/downloads/past-releases/elasticsearch-5-1-2](https://www.elastic.co/downloads/past-releases/elasticsearch-5-1-2) + Apache Unomi >= 1.3 : [https://www.elastic.co/downloads/past-releases/elasticsearch-5-6-3](https://www.elastic.co/downloads/past-releases/elasticsearch-5-6-3) + +2. Uncompress the downloaded package into a directory + +3. In the config/elasticsearch.yml file, uncomment and modify the following line : - bin/elasticsearch (Mac, Linux) - bin\elasticsearch.bat (Windows) + cluster.name: contextElasticSearch -3. Check that the ElasticSearch is up and running by accessing the following URL : +4. Launch the server using + + bin/elasticsearch (Mac, Linux) + bin\elasticsearch.bat (Windows) + +5. Check that the ElasticSearch is up and running by accessing the following URL : - http://localhost:9200 + [http://localhost:9200](http://localhost:9200) Deploying the generated binary package --------------------------------------
