This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 066c6d98a1961190f7a3693aa080e1d8eb815625 Author: Benoit Tellier <[email protected]> AuthorDate: Wed Sep 25 13:57:31 2019 +0700 JAMES-2563 Upgrade to ElasticSearch client 6.4.3 This enables health checks as part of the Rest High level client API, avoiding the need of low level hacky queries. --- backends-common/elasticsearch/pom.xml | 2 +- .../james/backends/es/ElasticSearchIndexer.java | 6 ++++-- .../james/backends/es/IndexCreationFactory.java | 8 +++++--- .../james/backends/es/NodeMappingFactory.java | 6 +++++- .../es/ClientProviderImplConnectionTest.java | 4 +++- .../james/backends/es/DockerElasticSearch.java | 3 +++ .../james/backends/es/DockerElasticSearchRule.java | 2 +- .../backends/es/ElasticSearchIndexerTest.java | 24 ++++++++++++++-------- .../backends/es/search/ScrolledSearchTest.java | 23 +++++++++++++-------- .../elasticsearch/ElasticSearchQuotaSearcher.java | 3 ++- 10 files changed, 55 insertions(+), 26 deletions(-) diff --git a/backends-common/elasticsearch/pom.xml b/backends-common/elasticsearch/pom.xml index f2ad34f..ea60bba 100644 --- a/backends-common/elasticsearch/pom.xml +++ b/backends-common/elasticsearch/pom.xml @@ -74,7 +74,7 @@ <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-high-level-client</artifactId> - <version>6.3.2</version> + <version>6.4.3</version> <exclusions> <exclusion> <groupId>commons-logging</groupId> diff --git a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchIndexer.java b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchIndexer.java index 1006232..bc586a4 100644 --- a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchIndexer.java +++ b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/ElasticSearchIndexer.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.xcontent.XContentType; @@ -72,7 +73,8 @@ public class ElasticSearchIndexer { new IndexRequest(aliasName.getValue()) .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) .id(id) - .source(content, XContentType.JSON)); + .source(content, XContentType.JSON), + RequestOptions.DEFAULT); } public Optional<BulkResponse> update(List<UpdatedRepresentation> updatedDocumentParts) throws IOException { @@ -84,7 +86,7 @@ public class ElasticSearchIndexer { NodeMappingFactory.DEFAULT_MAPPING_NAME, updatedDocumentPart.getId()) .doc(updatedDocumentPart.getUpdatedDocumentPart(), XContentType.JSON))); - return Optional.of(client.bulk(request)); + return Optional.of(client.bulk(request, RequestOptions.DEFAULT)); } catch (ValidationException e) { LOGGER.warn("Error while updating index", e); return Optional.empty(); diff --git a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/IndexCreationFactory.java b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/IndexCreationFactory.java index 9096fb8..18bfbf8 100644 --- a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/IndexCreationFactory.java +++ b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/IndexCreationFactory.java @@ -30,6 +30,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.xcontent.XContentBuilder; import org.slf4j.Logger; @@ -97,13 +98,14 @@ public class IndexCreationFactory { new IndicesAliasesRequest().addAliasAction( new AliasActions(AliasActions.Type.ADD) .index(indexName.getValue()) - .alias(aliasName.getValue()))); + .alias(aliasName.getValue())), + RequestOptions.DEFAULT); } } private boolean aliasExist(RestHighLevelClient client, AliasName aliasName) throws IOException { return client.indices() - .existsAlias(new GetAliasesRequest().aliases(aliasName.getValue())); + .existsAlias(new GetAliasesRequest().aliases(aliasName.getValue()), RequestOptions.DEFAULT); } private void createIndexIfNeeded(RestHighLevelClient client, IndexName indexName, XContentBuilder settings) throws IOException { @@ -111,7 +113,7 @@ public class IndexCreationFactory { client.indices() .create( new CreateIndexRequest(indexName.getValue()) - .source(settings)); + .source(settings), RequestOptions.DEFAULT); } catch (ElasticsearchStatusException exception) { if (exception.getMessage().contains(INDEX_ALREADY_EXISTS_EXCEPTION_MESSAGE)) { LOGGER.info("Index [{}] already exist", indexName); diff --git a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/NodeMappingFactory.java b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/NodeMappingFactory.java index 880d9ad..2dfa375 100644 --- a/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/NodeMappingFactory.java +++ b/backends-common/elasticsearch/src/main/java/org/apache/james/backends/es/NodeMappingFactory.java @@ -23,6 +23,7 @@ import java.io.IOException; import org.apache.http.HttpStatus; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -59,6 +60,8 @@ public class NodeMappingFactory { return client; } + // ElasticSearch 6.3.2 does not support field master_timeout that is set up by 6.4.3 REST client when relying on getMapping + @SuppressWarnings("deprecation") public static boolean mappingAlreadyExist(RestHighLevelClient client, IndexName indexName) throws IOException { try { client.getLowLevelClient().performRequest("GET", indexName.getValue() + "/_mapping/" + NodeMappingFactory.DEFAULT_MAPPING_NAME); @@ -75,7 +78,8 @@ public class NodeMappingFactory { client.indices().putMapping( new PutMappingRequest(indexName.getValue()) .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) - .source(mappingsSources)); + .source(mappingsSources), + RequestOptions.DEFAULT); } } diff --git a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ClientProviderImplConnectionTest.java b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ClientProviderImplConnectionTest.java index 3ae7449..042cbfd 100644 --- a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ClientProviderImplConnectionTest.java +++ b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ClientProviderImplConnectionTest.java @@ -26,6 +26,7 @@ import org.apache.james.util.docker.DockerContainer; import org.apache.james.util.docker.Images; import org.awaitility.Awaitility; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -97,7 +98,8 @@ public class ClientProviderImplConnectionTest { try (RestHighLevelClient client = clientProvider.get()) { client.search( new SearchRequest() - .source(new SearchSourceBuilder().query(QueryBuilders.existsQuery("any")))); + .source(new SearchSourceBuilder().query(QueryBuilders.existsQuery("any"))), + RequestOptions.DEFAULT); return true; } catch (Exception e) { LOGGER.info("Caught exception while trying to connect", e); diff --git a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearch.java b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearch.java index ffff560..9398dae 100644 --- a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearch.java +++ b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearch.java @@ -19,6 +19,9 @@ package org.apache.james.backends.es; +import java.time.Duration; +import java.util.Optional; + import org.apache.http.HttpStatus; import org.apache.james.util.Host; import org.apache.james.util.docker.DockerContainer; diff --git a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearchRule.java b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearchRule.java index a7de917..0f0d3d2 100644 --- a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearchRule.java +++ b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/DockerElasticSearchRule.java @@ -26,7 +26,7 @@ public class DockerElasticSearchRule extends ExternalResource { private final DockerElasticSearch dockerElasticSearch = DockerElasticSearchSingleton.INSTANCE; @Override - protected void before() throws Throwable { + protected void before() { dockerElasticSearch.start(); } diff --git a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchIndexerTest.java b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchIndexerTest.java index 08e4e4f..ba4e4f7 100644 --- a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchIndexerTest.java +++ b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/ElasticSearchIndexerTest.java @@ -31,6 +31,7 @@ import org.awaitility.Duration; import org.awaitility.core.ConditionFactory; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -82,7 +83,8 @@ public class ElasticSearchIndexerTest { SearchResponse searchResponse = client.search( new SearchRequest(INDEX_NAME.getValue()) - .source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("message", "trying")))); + .source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("message", "trying"))), + RequestOptions.DEFAULT); assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(1); } @@ -106,12 +108,14 @@ public class ElasticSearchIndexerTest { SearchResponse searchResponse = client.search( new SearchRequest(INDEX_NAME.getValue()) - .source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("message", "mastering")))); + .source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("message", "mastering"))), + RequestOptions.DEFAULT); assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(1); SearchResponse searchResponse2 = client.search( new SearchRequest(INDEX_NAME.getValue()) - .source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("field", "unchanged")))); + .source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("field", "unchanged"))), + RequestOptions.DEFAULT); assertThat(searchResponse2.getHits().getTotalHits()).isEqualTo(1); } @@ -153,7 +157,8 @@ public class ElasticSearchIndexerTest { CALMLY_AWAIT.atMost(Duration.TEN_SECONDS) .until(() -> client.search( new SearchRequest(INDEX_NAME.getValue()) - .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()))) + .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())), + RequestOptions.DEFAULT) .getHits().getTotalHits() == 0); } @@ -180,8 +185,9 @@ public class ElasticSearchIndexerTest { CALMLY_AWAIT.atMost(Duration.TEN_SECONDS) .until(() -> client.search( - new SearchRequest(INDEX_NAME.getValue()) - .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()))) + new SearchRequest(INDEX_NAME.getValue()) + .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())), + RequestOptions.DEFAULT) .getHits().getTotalHits() == 1); } @@ -198,7 +204,8 @@ public class ElasticSearchIndexerTest { SearchResponse searchResponse = client.search( new SearchRequest(INDEX_NAME.getValue()) - .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()))); + .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())), + RequestOptions.DEFAULT); assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(0); } @@ -225,7 +232,8 @@ public class ElasticSearchIndexerTest { SearchResponse searchResponse = client.search( new SearchRequest(INDEX_NAME.getValue()) - .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()))); + .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())), + RequestOptions.DEFAULT); assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(1); } diff --git a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/search/ScrolledSearchTest.java b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/search/ScrolledSearchTest.java index 49549af..6cd3e45 100644 --- a/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/search/ScrolledSearchTest.java +++ b/backends-common/elasticsearch/src/test/java/org/apache/james/backends/es/search/ScrolledSearchTest.java @@ -34,6 +34,7 @@ import org.awaitility.Duration; import org.awaitility.core.ConditionFactory; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; @@ -73,7 +74,7 @@ public class ScrolledSearchTest { } @Test - public void scrollIterableShouldWorkWhenEmpty() throws Exception { + public void scrollIterableShouldWorkWhenEmpty() { SearchRequest searchRequest = new SearchRequest(INDEX_NAME.getValue()) .scroll(TIMEOUT) .source(new SearchSourceBuilder() @@ -90,7 +91,8 @@ public class ScrolledSearchTest { client.index(new IndexRequest(INDEX_NAME.getValue()) .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) .id(id) - .source(MESSAGE, "Sample message")); + .source(MESSAGE, "Sample message"), + RequestOptions.DEFAULT); elasticSearch.awaitForElasticSearch(); WAIT_CONDITION.untilAsserted(() -> hasIdsInIndex(client, id)); @@ -112,13 +114,15 @@ public class ScrolledSearchTest { client.index(new IndexRequest(INDEX_NAME.getValue()) .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) .id(id1) - .source(MESSAGE, "Sample message")); + .source(MESSAGE, "Sample message"), + RequestOptions.DEFAULT); String id2 = "2"; client.index(new IndexRequest(INDEX_NAME.getValue()) .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) .id(id2) - .source(MESSAGE, "Sample message")); + .source(MESSAGE, "Sample message"), + RequestOptions.DEFAULT); elasticSearch.awaitForElasticSearch(); WAIT_CONDITION.untilAsserted(() -> hasIdsInIndex(client, id1, id2)); @@ -140,19 +144,22 @@ public class ScrolledSearchTest { client.index(new IndexRequest(INDEX_NAME.getValue()) .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) .id(id1) - .source(MESSAGE, "Sample message")); + .source(MESSAGE, "Sample message"), + RequestOptions.DEFAULT); String id2 = "2"; client.index(new IndexRequest(INDEX_NAME.getValue()) .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) .id(id2) - .source(MESSAGE, "Sample message")); + .source(MESSAGE, "Sample message"), + RequestOptions.DEFAULT); String id3 = "3"; client.index(new IndexRequest(INDEX_NAME.getValue()) .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) .id(id3) - .source(MESSAGE, "Sample message")); + .source(MESSAGE, "Sample message"), + RequestOptions.DEFAULT); elasticSearch.awaitForElasticSearch(); WAIT_CONDITION.untilAsserted(() -> hasIdsInIndex(client, id1, id2, id3)); @@ -174,7 +181,7 @@ public class ScrolledSearchTest { .source(new SearchSourceBuilder() .query(QueryBuilders.matchAllQuery())); - SearchHit[] hits = client.search(searchRequest) + SearchHit[] hits = client.search(searchRequest, RequestOptions.DEFAULT) .getHits() .getHits(); diff --git a/mailbox/plugin/quota-search-elasticsearch/src/main/java/org/apache/james/quota/search/elasticsearch/ElasticSearchQuotaSearcher.java b/mailbox/plugin/quota-search-elasticsearch/src/main/java/org/apache/james/quota/search/elasticsearch/ElasticSearchQuotaSearcher.java index 7c02032..1a7e314 100644 --- a/mailbox/plugin/quota-search-elasticsearch/src/main/java/org/apache/james/quota/search/elasticsearch/ElasticSearchQuotaSearcher.java +++ b/mailbox/plugin/quota-search-elasticsearch/src/main/java/org/apache/james/quota/search/elasticsearch/ElasticSearchQuotaSearcher.java @@ -34,6 +34,7 @@ import org.apache.james.core.User; import org.apache.james.quota.search.QuotaQuery; import org.apache.james.quota.search.QuotaSearcher; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; @@ -88,7 +89,7 @@ public class ElasticSearchQuotaSearcher implements QuotaSearcher { .types(NodeMappingFactory.DEFAULT_MAPPING_NAME) .source(searchSourceBuilder); - return Arrays.stream(client.search(searchRequest) + return Arrays.stream(client.search(searchRequest, RequestOptions.DEFAULT) .getHits() .getHits()); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
