Author: mkataria
Date: Tue Jul 28 14:53:46 2020
New Revision: 1880375
URL: http://svn.apache.org/viewvc?rev=1880375&view=rev
Log:
OAK-9151: Support term suggestion in Oak ES
Added:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexSuggestionCommonTest.java
(with props)
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticIndex.java.orig
(with props)
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticSuggestIterator.java
(with props)
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexSuggestionCommonTest.java
(with props)
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexSuggestionCommonTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticRequestHandler.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
Added:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexSuggestionCommonTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexSuggestionCommonTest.java?rev=1880375&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexSuggestionCommonTest.java
(added)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexSuggestionCommonTest.java
Tue Jul 28 14:53:46 2020
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.IndexSuggestionCommonTest;
+import org.apache.jackrabbit.oak.plugins.index.LuceneIndexOptions;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import java.io.File;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class LuceneIndexSuggestionCommonTest extends IndexSuggestionCommonTest
{
+ private ExecutorService executorService = Executors.newFixedThreadPool(2);
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder(new
File("target"));
+
+ @Override
+ protected Repository createJcrRepository() throws RepositoryException {
+ indexOptions = new LuceneIndexOptions();
+ repositoryOptionsUtil = new
LuceneTestRepositoryBuilder(executorService, temporaryFolder).build();
+ Oak oak = repositoryOptionsUtil.getOak();
+ Jcr jcr = new Jcr(oak);
+ Repository repository = jcr.createRepository();
+ return repository;
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexSuggestionCommonTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java?rev=1880375&r1=1880374&r2=1880375&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java
Tue Jul 28 14:53:46 2020
@@ -95,15 +95,15 @@ class ElasticDocument {
if (fulltext.size() > 0) {
builder.field(FieldNames.FULLTEXT, fulltext);
}
- if (suggest.size() > 0) {
- builder.startObject(FieldNames.SUGGEST).field("input",
suggest).endObject();
- }
- if (notNullProps.size() > 0) {
- builder.field(FieldNames.NOT_NULL_PROPS, notNullProps);
- }
- if (nullProps.size() > 0) {
- builder.field(FieldNames.NULL_PROPS, nullProps);
- }
+ if (suggest.size() > 0) {
+
builder.startObject(FieldNames.SUGGEST).field("suggestion",
suggest).endObject();
+ }
+ if (notNullProps.size() > 0) {
+ builder.field(FieldNames.NOT_NULL_PROPS, notNullProps);
+ }
+ if (nullProps.size() > 0) {
+ builder.field(FieldNames.NULL_PROPS, nullProps);
+ }
for (Map.Entry<String, Object> prop : properties.entrySet()) {
builder.field(prop.getKey(), prop.getValue());
}
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java?rev=1880375&r1=1880374&r2=1880375&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java
Tue Jul 28 14:53:46 2020
@@ -157,13 +157,31 @@ class ElasticIndexHelper {
Type<?> type = null;
boolean useInSpellCheck = false;
- for (PropertyDefinition pd: propertyDefinitions) {
+ boolean useInSuggest = false;
+ for (PropertyDefinition pd : propertyDefinitions) {
type = Type.fromTag(pd.getType(), false);
if (pd.useInSpellcheck) {
useInSpellCheck = true;
- break;
}
+ if (pd.useInSuggest) {
+ useInSuggest = true;
+ }
+ }
+ if (useInSuggest) {
+ mappingBuilder.startObject(FieldNames.SUGGEST);
+ {
+ mappingBuilder.field("type", "nested");
+ mappingBuilder.startObject("properties");
+ {
+ mappingBuilder.startObject("suggestion")
+ .field("type", "text")
+ .field("analyzer", "oak_analyzer")
+ .endObject();
+ }
+ mappingBuilder.endObject();
+ }
+ mappingBuilder.endObject();
}
mappingBuilder.startObject(name);
Added:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticIndex.java.orig
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticIndex.java.orig?rev=1880375&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticIndex.java.orig
(added)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticIndex.java.orig
Tue Jul 28 14:53:46 2020
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.elastic.query;
+
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticConnection;
+import
org.apache.jackrabbit.oak.plugins.index.elastic.query.async.ElasticResultRowAsyncIterator;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexNode;
+import org.apache.jackrabbit.oak.plugins.index.search.SizeEstimator;
+import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex;
+import
org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndexPlanner;
+import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator;
+import org.apache.jackrabbit.oak.spi.query.Cursor;
+import org.apache.jackrabbit.oak.spi.query.Filter;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.elasticsearch.common.Strings;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.WeakHashMap;
+import java.util.function.BiFunction;
+import java.util.function.Predicate;
+
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+import static
org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexDefinition.TYPE_ELASTICSEARCH;
+
+class ElasticIndex extends FulltextIndex {
+ private static final Predicate<NodeState>
ELASTICSEARCH_INDEX_DEFINITION_PREDICATE =
+ state ->
TYPE_ELASTICSEARCH.equals(state.getString(TYPE_PROPERTY_NAME));
+ private static final Map<String, LMSEstimator> ESTIMATORS = new
WeakHashMap<>();
+
+ // no concept of rewound in ES (even if it might be doing it internally,
we can't do much about it
+ private static final IteratorRewoundStateProvider
REWOUND_STATE_PROVIDER_NOOP = () -> 0;
+
+ // higher than some threshold below which the query should rather be
answered by something else if possible
+ private static final double MIN_COST = 100.1;
+
+ private final ElasticConnection elasticConnection;
+ private final NodeState root;
+
+ ElasticIndex(@NotNull ElasticConnection elasticConnection, @NotNull
NodeState root) {
+ this.elasticConnection = elasticConnection;
+ this.root = root;
+ }
+
+ @Override
+ protected String getType() {
+ return TYPE_ELASTICSEARCH;
+ }
+
+ @Override
+ protected FulltextIndexPlanner getPlanner(IndexNode indexNode, String
path, Filter filter, List<OrderEntry> sortOrder) {
+ return new ElasticIndexPlanner(indexNode, path, filter, sortOrder);
+ }
+
+ @Override
+ protected SizeEstimator getSizeEstimator(IndexPlan plan) {
+ return () ->
getEstimator(plan.getPlanName()).estimate(plan.getFilter());
+ }
+
+ @Override
+ protected Predicate<NodeState> getIndexDefinitionPredicate() {
+ return ELASTICSEARCH_INDEX_DEFINITION_PREDICATE;
+ }
+
+ @Override
+ public double getMinimumCost() {
+ return MIN_COST;
+ }
+
+ @Override
+ public String getIndexName() {
+ return "elasticsearch";
+ }
+
+ @Override
+ protected ElasticIndexNode acquireIndexNode(IndexPlan plan) {
+ return (ElasticIndexNode) super.acquireIndexNode(plan);
+ }
+
+ @Override
+ protected IndexNode acquireIndexNode(String indexPath) {
+ return new ElasticIndexNode(root, indexPath, elasticConnection);
+ }
+
+ @Override
+ protected String getFulltextRequestString(IndexPlan plan, IndexNode
indexNode) {
+ return Strings.toString(new ElasticRequestHandler(plan,
getPlanResult(plan)).baseQuery());
+ }
+
+ @Override
+ public Cursor query(IndexPlan plan, NodeState rootState) {
+ final Filter filter = plan.getFilter();
+ final FulltextIndexPlanner.PlanResult planResult = getPlanResult(plan);
+
+ final ElasticRequestHandler requestHandler = new
ElasticRequestHandler(plan, planResult);
+ final ElasticResponseHandler responseHandler = new
ElasticResponseHandler(planResult, filter);
+
+ final Iterator<FulltextResultRow> itr;
+ if (requestHandler.requiresSpellCheck()) {
+ itr = new ElasticSpellcheckIterator(acquireIndexNode(plan),
requestHandler, responseHandler);
+ } else {
+ // this function is called for each extracted row. Passing
FulltextIndex::shouldInclude means that for each
+ // row we evaluate getPathRestriction(plan) &
plan.getFilter().getPathRestriction(). Providing a partial
+ // function (https://en.wikipedia.org/wiki/Partial_function) we
can evaluate them once and still use a predicate as before
+ BiFunction<String, Filter.PathRestriction, Predicate<String>>
partialShouldInclude = (path, pathRestriction) -> docPath ->
+ shouldInclude(path, pathRestriction, docPath);
+
+ itr = new ElasticResultRowAsyncIterator(
+ acquireIndexNode(plan),
+ requestHandler,
+ responseHandler,
+ plan,
+ partialShouldInclude.apply(getPathRestriction(plan),
filter.getPathRestriction()),
+ getEstimator(plan.getPlanName())
+ );
+
+ }
+ return new FulltextPathCursor(itr, REWOUND_STATE_PROVIDER_NOOP, plan,
filter.getQueryLimits(), getSizeEstimator(plan));
+ }
+
+ private LMSEstimator getEstimator(String path) {
+ ESTIMATORS.putIfAbsent(path, new LMSEstimator());
+ return ESTIMATORS.get(path);
+ }
+
+ private static boolean shouldInclude(String path, Filter.PathRestriction
pathRestriction, String docPath) {
+ boolean include = true;
+ switch (pathRestriction) {
+ case EXACT:
+ include = path.equals(docPath);
+ break;
+ case DIRECT_CHILDREN:
+ include = PathUtils.getParentPath(docPath).equals(path);
+ break;
+ case ALL_CHILDREN:
+ include = PathUtils.isAncestor(path, docPath);
+ break;
+ }
+
+ return include;
+ }
+
+ @Override
+ protected boolean filterReplacedIndexes() {
+ return true;
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticIndex.java.orig
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticRequestHandler.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticRequestHandler.java?rev=1880375&r1=1880374&r2=1880375&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticRequestHandler.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticRequestHandler.java
Tue Jul 28 14:53:46 2020
@@ -39,10 +39,14 @@ import org.apache.jackrabbit.oak.spi.que
import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextTerm;
import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextVisitor;
import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.MatchBoolPrefixQueryBuilder;
import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
import org.elasticsearch.index.query.MoreLikeThisQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
+import org.elasticsearch.index.query.NestedQueryBuilder;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
@@ -59,6 +63,7 @@ import org.jetbrains.annotations.NotNull
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import javax.jcr.PropertyType;
import java.util.ArrayList;
import java.util.Arrays;
@@ -68,6 +73,7 @@ import java.util.concurrent.atomic.Atomi
import java.util.function.BiPredicate;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
+
import static org.apache.jackrabbit.JcrConstants.JCR_MIXINTYPES;
import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE;
import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot;
@@ -87,12 +93,13 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.spi.query.QueryConstants.JCR_PATH;
import static org.apache.jackrabbit.oak.spi.query.QueryConstants.JCR_SCORE;
import static org.apache.jackrabbit.util.ISO8601.parse;
+import static org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item;
/**
* Class to map query plans into Elastic request objects.
@@ -101,6 +108,7 @@ public class ElasticRequestHandler {
private static final Logger LOG =
LoggerFactory.getLogger(ElasticRequestHandler.class);
private final static String SPELLCHECK_PREFIX = "spellcheck?term=";
+ protected final static String SUGGEST_PREFIX = "suggest?term=";
private static final String ES_TRIGRAM_SUFFIX = ".trigram";
private static final List<FieldSortBuilder> DEFAULT_SORTS = Arrays.asList(
SortBuilders.fieldSort("_score").order(SortOrder.DESC),
@@ -210,6 +218,10 @@ public class ElasticRequestHandler {
return propertyRestrictionQuery != null &&
propertyRestrictionQuery.startsWith(SPELLCHECK_PREFIX);
}
+ public boolean requiresSuggestion() {
+ return propertyRestrictionQuery != null &&
propertyRestrictionQuery.startsWith(SUGGEST_PREFIX);
+ }
+
public ElasticFacetProvider getAsyncFacetProvider(ElasticResponseHandler
responseHandler) {
return requiresFacets() ?
ElasticFacetProvider.getProvider(
@@ -530,6 +542,17 @@ public class ElasticRequestHandler {
return queries;
}
+ public BoolQueryBuilder suggestionMatchQuery(String suggestion) {
+ QueryBuilder qb = new MatchBoolPrefixQueryBuilder(FieldNames.SUGGEST +
".suggestion", suggestion).operator(Operator.AND);
+ NestedQueryBuilder nestedQueryBuilder =
nestedQuery(FieldNames.SUGGEST, qb, ScoreMode.Max);
+ InnerHitBuilder in = new InnerHitBuilder().setSize(100);
+ nestedQueryBuilder.innerHit(in);
+ BoolQueryBuilder query = boolQuery()
+ .must(nestedQueryBuilder);
+ nonFullTextConstraints(indexPlan, planResult).forEach(query::must);
+ return query;
+ }
+
private static QueryBuilder
nodeTypeConstraints(IndexDefinition.IndexingRule defn, Filter filter) {
final BoolQueryBuilder bq = boolQuery();
PropertyDefinition primaryType = defn.getConfig(JCR_PRIMARYTYPE);
Added:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticSuggestIterator.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticSuggestIterator.java?rev=1880375&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticSuggestIterator.java
(added)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticSuggestIterator.java
Tue Jul 28 14:53:46 2020
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.elastic.query;
+
+import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
+import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
+import
org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex.FulltextResultRow;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.search.sort.SortOrder;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+/**
+ * This class is in charge to extract suggestions for a given query.
Suggestion is more like
+ * a completion result.
+ */
+class ElasticSuggestIterator implements Iterator<FulltextResultRow> {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(ElasticSuggestIterator.class);
+
+ private final ElasticIndexNode indexNode;
+ private final ElasticRequestHandler requestHandler;
+ private final ElasticResponseHandler responseHandler;
+ private final String suggestQuery;
+
+ private Iterator<? extends FulltextResultRow> internalIterator;
+ private boolean loaded;
+
+ ElasticSuggestIterator(@NotNull ElasticIndexNode indexNode,
+ @NotNull ElasticRequestHandler requestHandler,
+ @NotNull ElasticResponseHandler responseHandler) {
+ this.indexNode = indexNode;
+ this.requestHandler = requestHandler;
+ this.responseHandler = responseHandler;
+ this.suggestQuery =
requestHandler.getPropertyRestrictionQuery().replace(ElasticRequestHandler.SUGGEST_PREFIX,
"");
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (!loaded) {
+ try {
+ loadSuggestions();
+ } catch (IOException e) {
+ LOG.error("Failed loading suggestions", e);
+ throw new RuntimeException(e);
+ }
+ loaded = true;
+ }
+ return internalIterator != null && internalIterator.hasNext();
+ }
+
+ @Override
+ public FulltextResultRow next() {
+ return internalIterator.next();
+ }
+
+ private void loadSuggestions() throws IOException {
+ BoolQueryBuilder suggestionQuery =
requestHandler.suggestionMatchQuery(suggestQuery);
+ SearchSourceBuilder searchSourceBuilder =
SearchSourceBuilder.searchSource()
+ .query(suggestionQuery)
+ .size(100)
+ .fetchSource(FieldNames.PATH, null);
+ SearchRequest searchRequest = new
SearchRequest(indexNode.getDefinition().getRemoteIndexAlias())
+ .source(searchSourceBuilder);
+ SearchResponse res =
indexNode.getConnection().getClient().search(searchRequest,
RequestOptions.DEFAULT);
+ PriorityQueue<ElasticSuggestion> suggestionPriorityQueue = new
PriorityQueue<>((a, b) -> Double.compare(b.score, a.score));
+ for (SearchHit doc : res.getHits()) {
+ if (responseHandler.isAccessible(responseHandler.getPath(doc))) {
+ for (SearchHit suggestion :
doc.getInnerHits().get(FieldNames.SUGGEST).getHits()) {
+ suggestionPriorityQueue.add(new
ElasticSuggestion(((List<String>)
suggestion.getSourceAsMap().get("suggestion")).get(0), suggestion.getScore()));
+ }
+ }
+ }
+ this.internalIterator = suggestionPriorityQueue.iterator();
+ }
+
+ private final static class ElasticSuggestion extends FulltextResultRow{
+ private ElasticSuggestion(String suggestion, double score) {
+ super(suggestion, score);
+ }
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticSuggestIterator.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexSuggestionCommonTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexSuggestionCommonTest.java?rev=1880375&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexSuggestionCommonTest.java
(added)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexSuggestionCommonTest.java
Tue Jul 28 14:53:46 2020
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.elastic;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.ElasticTestRepositoryBuilder;
+import org.apache.jackrabbit.oak.plugins.index.IndexSuggestionCommonTest;
+import org.apache.jackrabbit.oak.plugins.index.TestUtils;
+import org.junit.After;
+import org.junit.ClassRule;
+
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import java.io.IOException;
+
+public class ElasticIndexSuggestionCommonTest extends
IndexSuggestionCommonTest {
+
+ // Set this connection string as
+ // <scheme>://<hostname>:<port>?key_id=<>,key_secret=<>
+ // key_id and key_secret are optional in case the ES server
+ // needs authentication
+ // Do not set this if docker is running and you want to run the tests on
docker instead.
+ private static String elasticConnectionString =
System.getProperty("elasticConnectionString");
+ @ClassRule
+ public static ElasticConnectionRule elasticRule = new
ElasticConnectionRule(elasticConnectionString);
+
+ /*
+ Close the ES connection after every test method execution
+ */
+ @After
+ public void cleanup() throws IOException {
+ elasticRule.closeElasticConnection();
+ }
+
+ protected Repository createJcrRepository() throws RepositoryException {
+ indexOptions = new ElasticIndexOptions();
+ repositoryOptionsUtil = new
ElasticTestRepositoryBuilder(elasticRule).build();
+ Oak oak = repositoryOptionsUtil.getOak();
+ Jcr jcr = new Jcr(oak);
+ Repository repository = jcr.createRepository();
+ return repository;
+ }
+
+ protected void assertEventually(Runnable r) {
+ TestUtils.assertEventually(r,
+ ((repositoryOptionsUtil.isAsync() ?
repositoryOptionsUtil.defaultAsyncIndexingTimeInSeconds : 0) +
ElasticIndexDefinition.BULK_FLUSH_INTERVAL_MS_DEFAULT) * 5);
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexSuggestionCommonTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java?rev=1880375&r1=1880374&r2=1880375&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
Tue Jul 28 14:53:46 2020
@@ -341,6 +341,10 @@ public abstract class FulltextIndex impl
}
public FulltextResultRow(String suggestion, long weight) {
+ this(suggestion, (double)weight);
+ }
+
+ public FulltextResultRow(String suggestion, double weight) {
this.isVirutal = true;
this.path = "/";
this.score = weight;
Added:
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexSuggestionCommonTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexSuggestionCommonTest.java?rev=1880375&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexSuggestionCommonTest.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexSuggestionCommonTest.java
Tue Jul 28 14:53:46 2020
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index;
+
+import org.apache.jackrabbit.JcrConstants;
+import org.apache.jackrabbit.api.JackrabbitSession;
+import
org.apache.jackrabbit.commons.jackrabbit.authorization.AccessControlUtils;
+import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexFormatVersion;
+import org.apache.jackrabbit.oak.query.AbstractJcrTest;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.jcr.Node;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+import javax.jcr.query.Row;
+import javax.jcr.query.RowIterator;
+import javax.jcr.security.Privilege;
+
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_RULES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public abstract class IndexSuggestionCommonTest extends AbstractJcrTest {
+ protected TestRepository repositoryOptionsUtil;
+ protected Node indexNode;
+ protected IndexOptions indexOptions;
+
+ private JackrabbitSession session = null;
+ private Node root = null;
+
+ @Before
+ public void settingup() throws RepositoryException {
+ session = (JackrabbitSession) adminSession;
+ root = session.getRootNode();
+ }
+
+ private void createSuggestIndex(String name, String indexedNodeType,
String indexedPropertyName)
+ throws Exception {
+ createSuggestIndex(name, indexedNodeType, indexedPropertyName, false,
false);
+ }
+
+ private void createSuggestIndex(String name, String indexedNodeType,
String indexedPropertyName, boolean addFullText, boolean suggestAnalyzed)
+ throws Exception {
+ Node def = root.getNode(INDEX_DEFINITIONS_NAME)
+ .addNode(name, INDEX_DEFINITIONS_NODE_TYPE);
+ def.setProperty(TYPE_PROPERTY_NAME, indexOptions.getIndexType());
+ def.setProperty(REINDEX_PROPERTY_NAME, true);
+ def.setProperty("name", name);
+ def.setProperty(FulltextIndexConstants.COMPAT_MODE,
IndexFormatVersion.V2.getVersion());
+ if (suggestAnalyzed) {
+
def.addNode(FulltextIndexConstants.SUGGESTION_CONFIG).setProperty("suggestAnalyzed",
suggestAnalyzed);
+ }
+
+
+ Node propertyIdxDef = def.addNode(INDEX_RULES,
JcrConstants.NT_UNSTRUCTURED)
+ .addNode(indexedNodeType, JcrConstants.NT_UNSTRUCTURED)
+ .addNode(FulltextIndexConstants.PROP_NODE,
JcrConstants.NT_UNSTRUCTURED)
+ .addNode("indexedProperty", JcrConstants.NT_UNSTRUCTURED);
+ propertyIdxDef.setProperty("propertyIndex", true);
+ propertyIdxDef.setProperty("analyzed", true);
+ propertyIdxDef.setProperty("useInSuggest", true);
+ if (addFullText) {
+ propertyIdxDef.setProperty("nodeScopeIndex", true);
+ }
+ propertyIdxDef.setProperty("name", indexedPropertyName);
+ }
+
+ /**
+ * Utility method to check suggestion over {@code nodeType} when the index
definition also created for
+ * the same type
+ */
+ private void checkSuggestions(final String nodeType,
+ final String indexPropName, final String
indexPropValue,
+ final boolean addFullText, final boolean
useUserSession,
+ final String suggestQueryText, final boolean
shouldSuggest, final boolean suggestAnalyzed)
+ throws Exception {
+ checkSuggestions(nodeType, nodeType,
+ indexPropName, indexPropValue,
+ addFullText, useUserSession,
+ suggestQueryText, shouldSuggest, suggestAnalyzed);
+ }
+
+ /**
+ * Utility method to check suggestion over {@code queryNodeType} when the
index definition is created for
+ * {@code indexNodeType}
+ */
+ private void checkSuggestions(final String indexNodeType, final String
queryNodeType,
+ final String indexPropName, final String
indexPropValue,
+ final boolean addFullText, final boolean
useUserSession,
+ final String suggestQueryText, final boolean
shouldSuggest,
+ final boolean suggestAnalyzed)
+ throws Exception {
+ createSuggestIndex("lucene-suggest", indexNodeType, indexPropName,
addFullText, suggestAnalyzed);
+
+ Node indexedNode = root.addNode("indexedNode1", queryNodeType);
+
+ if (indexPropValue != null) {
+ indexedNode.setProperty(indexPropName, indexPropValue + " 1");
+ indexedNode = root.addNode("indexedNode2", queryNodeType);
+ indexedNode.setProperty(indexPropName, indexPropValue + " 2");
+ }
+
+ session.save();
+
+ Session userSession = session;
+
+ if (useUserSession) {
+ allow(indexedNode);
+ session.save();
+ userSession = anonymousSession;//repository.login(new
SimpleCredentials(TEST_USER_NAME, TEST_USER_NAME.toCharArray()));
+ }
+
+ String suggQuery = createSuggestQuery(queryNodeType, suggestQueryText);
+ QueryManager queryManager =
userSession.getWorkspace().getQueryManager();
+ if (shouldSuggest) {
+ assertEventually(() -> {
+ try {
+ assertNotNull("There should be some suggestion",
getResult(queryManager, suggQuery));
+ } catch (RepositoryException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ } else {
+ assertEventually(() -> {
+ try {
+ assertNull("There shouldn't be any suggestion",
getResult(queryManager, suggQuery));
+ } catch (RepositoryException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ }
+ userSession.logout();
+ }
+
+ private String getResult(QueryManager queryManager, String suggQuery)
throws RepositoryException {
+
+ QueryResult result = queryManager.createQuery(suggQuery,
Query.JCR_SQL2).execute();
+ RowIterator rows = result.getRows();
+
+ String value = null;
+ while (rows.hasNext()) {
+ Row firstRow = rows.nextRow();
+ value = firstRow.getValue("suggestion").getString();
+ break;
+ }
+ return value;
+ }
+
+ private Node allow(Node node) throws RepositoryException {
+ AccessControlUtils.allow(node, "anonymous", Privilege.JCR_READ);
+ return node;
+ }
+
+ private String createSuggestQuery(String nodeTypeName, String suggestFor) {
+ return "SELECT [rep:suggest()] as suggestion, [jcr:score] as score
FROM [" + nodeTypeName + "] WHERE suggest('" + suggestFor + "')";
+ }
+
+ @Test
+ public void suggestNodeName() throws Exception {
+ final String nodeType = "nt:unstructured";
+
+ createSuggestIndex("lucene-suggest", nodeType,
FulltextIndexConstants.PROPDEF_PROP_NODE_NAME);
+
+ root.addNode("indexedNode", nodeType);
+ adminSession.save();
+
+ String suggQuery = createSuggestQuery(nodeType, "indexedn");
+ QueryManager queryManager =
adminSession.getWorkspace().getQueryManager();
+ assertEventually(() -> {
+ try {
+ assertEquals("Node name should be suggested", "indexedNode",
getResult(queryManager, suggQuery));
+ } catch (RepositoryException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ //OAK-3157
+ @Test
+ public void testSuggestQuery() throws Exception {
+ final String nodeType = "nt:unstructured";
+ final String indexPropName = "description";
+ final String indexPropValue = "this is just a sample text which should
get some response in suggestions";
+ final String suggestQueryText = "th";
+ final boolean shouldSuggest = true;
+
+ checkSuggestions(nodeType,
+ indexPropName, indexPropValue,
+ false, false,
+ suggestQueryText, shouldSuggest, false);
+ }
+
+ //OAK-4126
+ @Test
+ public void testSuggestQuerySpecialChars() throws Exception {
+ final String nodeType = "nt:unstructured";
+ final String indexPropName = "description";
+ final String indexPropValue = "DD~@#$%^&*()_+{}\":?><`1234567890-=[]";
+ final String suggestQueryText = "dd";
+ final boolean shouldSuggest = true;
+
+ checkSuggestions(nodeType,
+ indexPropName, indexPropValue,
+ false, false,
+ suggestQueryText, shouldSuggest, false);
+ }
+
+ @Test
+ public void avoidInfiniteSuggestions() throws Exception {
+ final String nodeType = "nt:unstructured";
+ final String indexPropName = "description";
+ final String higherRankPropValue = "DD DD DD DD";
+ final String exceptionThrowingPropValue =
"DD~@#$%^&*()_+{}\":?><`1234567890-=[]";
+ final String suggestQueryText = "dd";
+
+ createSuggestIndex("lucene-suggest", nodeType, indexPropName);
+
+ root.addNode("higherRankNode", nodeType).setProperty(indexPropName,
higherRankPropValue);
+ root.addNode("exceptionThrowingNode",
nodeType).setProperty(indexPropName, exceptionThrowingPropValue);
+ adminSession.save();
+
+ String suggQuery = createSuggestQuery(nodeType, suggestQueryText);
+ QueryManager queryManager =
adminSession.getWorkspace().getQueryManager();
+ QueryResult result = queryManager.createQuery(suggQuery,
Query.JCR_SQL2).execute();
+ RowIterator rows = result.getRows();
+
+ int count = 0;
+ while (count < 3 && rows.hasNext()) {
+ count++;
+ rows.nextRow();
+ }
+
+ assertTrue("There must not be more than 2 suggestions", count <= 2);
+ }
+
+ //OAK-3156
+ @Test
+ public void testSuggestQueryWithUserAccess() throws Exception {
+ final String nodeType = "nt:unstructured";
+ final String indexPropName = "description";
+ final String indexPropValue = "this is just a sample text which should
get some response in suggestions";
+ final String suggestQueryText = "this is jus";
+ final boolean shouldSuggest = true;
+
+ checkSuggestions(nodeType,
+ indexPropName, indexPropValue,
+ false, true,
+ suggestQueryText, shouldSuggest, false);
+ }
+
+ //OAK-3156
+ @Test
+ public void testSuggestQueryFromMoreGeneralNodeType() throws Exception {
+ final String indexNodeType = "nt:base";
+ final String queryNodeType = "oak:Unstructured";
+ final String indexPropName = "description";
+ final String indexPropValue = "this is just a sample text which should
get some response in suggestions";
+ final String suggestQueryText = "th";
+ final boolean shouldSuggest = false;
+
+ checkSuggestions(indexNodeType, queryNodeType,
+ indexPropName, indexPropValue,
+ true, false,
+ suggestQueryText, shouldSuggest, false);
+ }
+
+ //OAK-3156
+ @Test
+ public void testSuggestQueryOnNonNtBase() throws Exception {
+ final String nodeType = "oak:Unstructured";
+ final String indexPropName = "description";
+ final String indexPropValue = "this is just a sample text which should
get some response in suggestions";
+ final String suggestQueryText = "th";
+ final boolean shouldSuggest = true;
+
+ checkSuggestions(nodeType,
+ indexPropName, indexPropValue,
+ true, false,
+ suggestQueryText, shouldSuggest, false);
+ }
+
+ //OAK-3509
+ @Test
+ public void testMultipleSuggestions() throws Exception {
+ final String nodeType = "oak:Unstructured";
+ final String indexPropName = "description";
+ final String indexPropValue = "this is just a sample text which should
get some response in suggestions";
+ final String suggestQueryText = "th";
+ final boolean shouldSuggest = true;
+
+ checkSuggestions(nodeType,
+ indexPropName, indexPropValue,
+ true, false,
+ suggestQueryText, shouldSuggest, false);
+ }
+
+ //OAK-3407
+ @Test
+ public void testSuggestQueryAnalyzed() throws Exception {
+ final String nodeType = "nt:unstructured";
+ final String indexPropName = "description";
+ final String indexPropValue = "this is just a sample text which should
get some response in suggestions";
+ final String suggestQueryText = "sa";
+
+ checkSuggestions(nodeType,
+ indexPropName, indexPropValue,
+ true, true,
+ suggestQueryText, true, true);
+ }
+
+ //OAK-3149
+ @Test
+ public void testSuggestQueryInfix() throws Exception {
+ final String nodeType = "nt:unstructured";
+ final String indexPropName = "description";
+ final String indexPropValue = "this is just a sample text which should
get some response in suggestions";
+ final String suggestQueryText = "sa";
+
+ checkSuggestions(nodeType,
+ indexPropName, indexPropValue,
+ true, true,
+ suggestQueryText, true, false);
+ }
+
+ //OAK-4067
+ @Test
+ public void emptySuggestWithNothingIndexed() throws Exception {
+ final String nodeType = "nt:unstructured";
+ final String indexPropName = "description";
+ final String indexPropValue = null;
+ final String suggestQueryText = null;
+
+ checkSuggestions(nodeType,
+ indexPropName, indexPropValue,
+ true, true,
+ suggestQueryText, false, false);
+ }
+
+ private static void assertEventually(Runnable r) {
+ TestUtils.assertEventually(r, 3000 * 3);
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexSuggestionCommonTest.java
------------------------------------------------------------------------------
svn:eol-style = native
