This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 0ad1f0d061b73c584f73f65bc84847af49109e18 Author: Benoit Tellier <[email protected]> AuthorDate: Fri May 17 11:58:02 2019 +0700 JAMES-2765 Type naming is no more supported in ES 6 --- .../james/backends/es/v6/ElasticSearchIndexer.java | 23 +++++++--------- .../james/backends/es/v6/NodeMappingFactory.java | 12 ++++---- .../org/apache/james/backends/es/v6/TypeName.java | 32 ---------------------- .../backends/es/v6/DockerElasticSearchRule.java | 4 +++ .../backends/es/v6/ElasticSearchIndexerTest.java | 10 +------ .../backends/es/v6/NodeMappingFactoryTest.java | 14 +++------- .../backends/es/v6/search/ScrollIterableTest.java | 32 +++++++++++++--------- 7 files changed, 44 insertions(+), 83 deletions(-) diff --git a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/ElasticSearchIndexer.java b/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/ElasticSearchIndexer.java index 79adb69..9689c1a 100644 --- a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/ElasticSearchIndexer.java +++ b/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/ElasticSearchIndexer.java @@ -51,23 +51,19 @@ public class ElasticSearchIndexer { private final RestHighLevelClient client; private final AliasName aliasName; - private final TypeName typeName; private final int batchSize; public ElasticSearchIndexer(RestHighLevelClient client, - WriteAliasName aliasName, - TypeName typeName) { - this(client, aliasName, typeName, DEFAULT_BATCH_SIZE); + WriteAliasName aliasName) { + this(client, aliasName, DEFAULT_BATCH_SIZE); } @VisibleForTesting public ElasticSearchIndexer(RestHighLevelClient client, WriteAliasName aliasName, - TypeName typeName, int batchSize) { this.client = client; this.aliasName = aliasName; - this.typeName = typeName; this.batchSize = batchSize; } @@ -77,7 +73,9 @@ public class ElasticSearchIndexer { LOGGER.debug("Indexing {}: {}", id, StringUtils.left(content, DEBUG_MAX_LENGTH_CONTENT)); } return client.index( - new IndexRequest(aliasName.getValue(), typeName.getValue(), id) + new IndexRequest(aliasName.getValue()) + .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) + .id(id) .source(content, XContentType.JSON), RequestOptions.DEFAULT); } @@ -88,7 +86,7 @@ public class ElasticSearchIndexer { BulkRequest request = new BulkRequest(); updatedDocumentParts.forEach(updatedDocumentPart -> request.add( new UpdateRequest(aliasName.getValue(), - typeName.getValue(), + NodeMappingFactory.DEFAULT_MAPPING_NAME, updatedDocumentPart.getId()) .doc(updatedDocumentPart.getUpdatedDocumentPart(), XContentType.JSON))); return Optional.of(client.bulk(request, RequestOptions.DEFAULT)); @@ -102,10 +100,9 @@ public class ElasticSearchIndexer { try { BulkRequest request = new BulkRequest(); ids.forEach(id -> request.add( - new DeleteRequest( - aliasName.getValue(), - typeName.getValue(), - id))); + new DeleteRequest(aliasName.getValue()) + .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) + .id(id))); return Optional.of(client.bulk(request, RequestOptions.DEFAULT)); } catch (ValidationException e) { LOGGER.warn("Error while deleting index", e); @@ -115,7 +112,7 @@ public class ElasticSearchIndexer { public void deleteAllMatchingQuery(QueryBuilder queryBuilder) { DeleteByQueryRequest request = new DeleteByQueryRequest(aliasName.getValue()) - .setDocTypes(typeName.getValue()) + .setDocTypes(NodeMappingFactory.DEFAULT_MAPPING_NAME) .setScroll(TIMEOUT) .setQuery(queryBuilder) .setBatchSize(batchSize); diff --git a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/NodeMappingFactory.java b/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/NodeMappingFactory.java index eda3c50..237f7fa 100644 --- a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/NodeMappingFactory.java +++ b/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/NodeMappingFactory.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; public class NodeMappingFactory { + public static final String DEFAULT_MAPPING_NAME = "_doc"; public static final String BOOLEAN = "boolean"; public static final String TYPE = "type"; public static final String LONG = "long"; @@ -38,6 +39,7 @@ public class NodeMappingFactory { public static final String NOT_ANALYZED = "not_analyzed"; public static final String STRING = "string"; public static final String TEXT = "text"; + public static final String KEYWORD = "keyword"; public static final String PROPERTIES = "properties"; public static final String DATE = "date"; public static final String FORMAT = "format"; @@ -50,14 +52,14 @@ public class NodeMappingFactory { public static final String SNOWBALL = "snowball"; public static final String IGNORE_ABOVE = "ignore_above"; - public static RestHighLevelClient applyMapping(RestHighLevelClient client, IndexName indexName, TypeName typeName, XContentBuilder mappingsSources) throws IOException { - if (!mappingAlreadyExist(client, indexName, typeName)) { + public static RestHighLevelClient applyMapping(RestHighLevelClient client, IndexName indexName, XContentBuilder mappingsSources) throws IOException { + if (!mappingAlreadyExist(client, indexName)) { createMapping(client, indexName, mappingsSources); } return client; } - public static boolean mappingAlreadyExist(RestHighLevelClient client, IndexName indexName, TypeName typeName) throws IOException { + public static boolean mappingAlreadyExist(RestHighLevelClient client, IndexName indexName) throws IOException { return Iterators.toStream(client.indices() .getMapping( new GetMappingsRequest() @@ -66,12 +68,10 @@ public class NodeMappingFactory { .mappings() .values() .iterator()) - .anyMatch(mapping -> mapping.type().contains(typeName.getValue())); + .anyMatch(mappingMetaData -> !mappingMetaData.getSourceAsMap().isEmpty()); } public static void createMapping(RestHighLevelClient client, IndexName indexName, XContentBuilder mappingsSources) throws IOException { - PutMappingRequest request = new PutMappingRequest(indexName.getValue()) - .source(mappingsSources); client.indices().putMapping( new PutMappingRequest(indexName.getValue()) .source(mappingsSources), diff --git a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/TypeName.java b/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/TypeName.java deleted file mode 100644 index 7f3dbf7..0000000 --- a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/TypeName.java +++ /dev/null @@ -1,32 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ - -package org.apache.james.backends.es.v6; - -public class TypeName { - private final String value; - - public TypeName(String value) { - this.value = value; - } - - public String getValue() { - return value; - } -} diff --git a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/DockerElasticSearchRule.java b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/DockerElasticSearchRule.java index e945e31..08d891b 100644 --- a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/DockerElasticSearchRule.java +++ b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/DockerElasticSearchRule.java @@ -42,4 +42,8 @@ public class DockerElasticSearchRule extends ExternalResource { public void awaitForElasticSearch() { dockerElasticSearch.awaitForElasticSearch(); } + + public DockerElasticSearch getDockerElasticSearch() { + return dockerElasticSearch; + } } diff --git a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/ElasticSearchIndexerTest.java b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/ElasticSearchIndexerTest.java index 08f02a2..3ca61c1 100644 --- a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/ElasticSearchIndexerTest.java +++ b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/ElasticSearchIndexerTest.java @@ -42,7 +42,6 @@ public class ElasticSearchIndexerTest { private static final int MINIMUM_BATCH_SIZE = 1; private static final IndexName INDEX_NAME = new IndexName("index_name"); private static final WriteAliasName ALIAS_NAME = new WriteAliasName("alias_name"); - private static final TypeName TYPE_NAME = new TypeName("type_name"); @Rule public DockerElasticSearchRule elasticSearch = new DockerElasticSearchRule(); @@ -54,7 +53,7 @@ public class ElasticSearchIndexerTest { .useIndex(INDEX_NAME) .addAlias(ALIAS_NAME) .createIndexAndAliases(getESClient()); - testee = new ElasticSearchIndexer(getESClient(), ALIAS_NAME, TYPE_NAME, MINIMUM_BATCH_SIZE); + testee = new ElasticSearchIndexer(getESClient(), ALIAS_NAME, MINIMUM_BATCH_SIZE); } private RestHighLevelClient getESClient() { @@ -72,7 +71,6 @@ public class ElasticSearchIndexerTest { try (RestHighLevelClient client = getESClient()) { SearchResponse searchResponse = client.search( new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("message", "trying"))), RequestOptions.DEFAULT); assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(1); @@ -99,7 +97,6 @@ public class ElasticSearchIndexerTest { try (RestHighLevelClient client = getESClient()) { SearchResponse searchResponse = client.search( new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("message", "mastering"))), RequestOptions.DEFAULT); assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(1); @@ -108,7 +105,6 @@ public class ElasticSearchIndexerTest { try (RestHighLevelClient client = getESClient()) { SearchResponse searchResponse = client.search( new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("field", "unchanged"))), RequestOptions.DEFAULT); assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(1); @@ -154,7 +150,6 @@ public class ElasticSearchIndexerTest { await().atMost(Duration.TEN_SECONDS) .until(() -> client.search( new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())), RequestOptions.DEFAULT) .getHits().getTotalHits() == 0); @@ -186,7 +181,6 @@ public class ElasticSearchIndexerTest { await().atMost(Duration.TEN_SECONDS) .until(() -> client.search( new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())), RequestOptions.DEFAULT) .getHits().getTotalHits() == 1); @@ -207,7 +201,6 @@ public class ElasticSearchIndexerTest { try (RestHighLevelClient client = getESClient()) { SearchResponse searchResponse = client.search( new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())), RequestOptions.DEFAULT); assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(0); @@ -238,7 +231,6 @@ public class ElasticSearchIndexerTest { try (RestHighLevelClient client = getESClient()) { SearchResponse searchResponse = client.search( new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())), RequestOptions.DEFAULT); assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(1); diff --git a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/NodeMappingFactoryTest.java b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/NodeMappingFactoryTest.java index 95207ff..86fad09 100644 --- a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/NodeMappingFactoryTest.java +++ b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/NodeMappingFactoryTest.java @@ -19,10 +19,9 @@ package org.apache.james.backends.es.v6; -import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.common.xcontent.XContentBuilder; import org.junit.Before; import org.junit.Rule; @@ -32,7 +31,6 @@ public class NodeMappingFactoryTest { private static final String MESSAGE = "message"; private static final IndexName INDEX_NAME = new IndexName("index"); private static final ReadAliasName ALIAS_NAME = new ReadAliasName("alias"); - private static final TypeName TYPE_NAME = new TypeName("type"); @Rule public DockerElasticSearchRule elasticSearch = new DockerElasticSearchRule(); @@ -47,7 +45,6 @@ public class NodeMappingFactoryTest { .createIndexAndAliases(clientProvider.get()); NodeMappingFactory.applyMapping(clientProvider.get(), INDEX_NAME, - TYPE_NAME, getMappingsSources()); } @@ -55,24 +52,21 @@ public class NodeMappingFactoryTest { public void applyMappingShouldNotThrowWhenCalledSeveralTime() throws Exception { NodeMappingFactory.applyMapping(clientProvider.get(), INDEX_NAME, - TYPE_NAME, getMappingsSources()); } @Test - public void applyMappingShouldThrowWhenTryingIndexerChanges() throws Exception { + public void applyMappingShouldNotThrowWhenIncrementalChanges() throws Exception { NodeMappingFactory.applyMapping(clientProvider.get(), INDEX_NAME, - TYPE_NAME, getMappingsSources()); elasticSearch.awaitForElasticSearch(); - assertThatThrownBy(() ->NodeMappingFactory.applyMapping(clientProvider.get(), + assertThatCode(() ->NodeMappingFactory.applyMapping(clientProvider.get(), INDEX_NAME, - TYPE_NAME, getOtherMappingsSources())) - .isInstanceOf(ElasticsearchStatusException.class); + .doesNotThrowAnyException(); } private XContentBuilder getMappingsSources() throws Exception { diff --git a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/search/ScrollIterableTest.java b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/search/ScrollIterableTest.java index c9709b1..3840181 100644 --- a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/search/ScrollIterableTest.java +++ b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/search/ScrollIterableTest.java @@ -32,8 +32,8 @@ import org.apache.james.backends.es.v6.DockerElasticSearchRule; import org.apache.james.backends.es.v6.ElasticSearchConfiguration; import org.apache.james.backends.es.v6.IndexCreationFactory; import org.apache.james.backends.es.v6.IndexName; +import org.apache.james.backends.es.v6.NodeMappingFactory; import org.apache.james.backends.es.v6.ReadAliasName; -import org.apache.james.backends.es.v6.TypeName; import org.awaitility.Duration; import org.awaitility.core.ConditionFactory; import org.elasticsearch.action.index.IndexRequest; @@ -55,7 +55,6 @@ public class ScrollIterableTest { private static final String MESSAGE = "message"; private static final IndexName INDEX_NAME = new IndexName("index"); private static final ReadAliasName ALIAS_NAME = new ReadAliasName("alias"); - private static final TypeName TYPE_NAME = new TypeName("messages"); private static final ConditionFactory WAIT_CONDITION = await().timeout(Duration.FIVE_SECONDS); @@ -77,7 +76,6 @@ public class ScrollIterableTest { public void scrollIterableShouldWorkWhenEmpty() throws Exception { try (RestHighLevelClient client = clientProvider.get()) { SearchRequest searchRequest = new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .scroll(TIMEOUT) .source(new SearchSourceBuilder() .query(QueryBuilders.matchAllQuery()) @@ -92,7 +90,9 @@ public class ScrollIterableTest { public void scrollIterableShouldWorkWhenOneElement() throws Exception { try (RestHighLevelClient client = clientProvider.get()) { String id = "1"; - client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id) + client.index(new IndexRequest(INDEX_NAME.getValue()) + .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) + .id(id) .source(MESSAGE, "Sample message"), RequestOptions.DEFAULT); @@ -100,7 +100,6 @@ public class ScrollIterableTest { WAIT_CONDITION.untilAsserted(() -> hasIdsInIndex(client, id)); SearchRequest searchRequest = new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .scroll(TIMEOUT) .source(new SearchSourceBuilder() .query(QueryBuilders.matchAllQuery()) @@ -115,12 +114,16 @@ public class ScrollIterableTest { public void scrollIterableShouldWorkWhenSizeElement() throws Exception { try (RestHighLevelClient client = clientProvider.get()) { String id1 = "1"; - client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id1) + client.index(new IndexRequest(INDEX_NAME.getValue()) + .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) + .id(id1) .source(MESSAGE, "Sample message"), RequestOptions.DEFAULT); String id2 = "2"; - client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id2) + client.index(new IndexRequest(INDEX_NAME.getValue()) + .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) + .id(id2) .source(MESSAGE, "Sample message"), RequestOptions.DEFAULT); @@ -128,7 +131,6 @@ public class ScrollIterableTest { WAIT_CONDITION.untilAsserted(() -> hasIdsInIndex(client, id1, id2)); SearchRequest searchRequest = new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .scroll(TIMEOUT) .source(new SearchSourceBuilder() .query(QueryBuilders.matchAllQuery()) @@ -143,17 +145,23 @@ public class ScrollIterableTest { public void scrollIterableShouldWorkWhenMoreThanSizeElement() throws Exception { try (RestHighLevelClient client = clientProvider.get()) { String id1 = "1"; - client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id1) + client.index(new IndexRequest(INDEX_NAME.getValue()) + .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) + .id(id1) .source(MESSAGE, "Sample message"), RequestOptions.DEFAULT); String id2 = "2"; - client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id2) + client.index(new IndexRequest(INDEX_NAME.getValue()) + .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) + .id(id2) .source(MESSAGE, "Sample message"), RequestOptions.DEFAULT); String id3 = "3"; - client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id3) + client.index(new IndexRequest(INDEX_NAME.getValue()) + .type(NodeMappingFactory.DEFAULT_MAPPING_NAME) + .id(id3) .source(MESSAGE, "Sample message"), RequestOptions.DEFAULT); @@ -161,7 +169,6 @@ public class ScrollIterableTest { WAIT_CONDITION.untilAsserted(() -> hasIdsInIndex(client, id1, id2, id3)); SearchRequest searchRequest = new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .scroll(TIMEOUT) .source(new SearchSourceBuilder() .query(QueryBuilders.matchAllQuery()) @@ -181,7 +188,6 @@ public class ScrollIterableTest { private void hasIdsInIndex(RestHighLevelClient client, String... ids) throws IOException { SearchRequest searchRequest = new SearchRequest(INDEX_NAME.getValue()) - .types(TYPE_NAME.getValue()) .scroll(TIMEOUT) .source(new SearchSourceBuilder() .query(QueryBuilders.matchAllQuery())); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
