Author: tommaso
Date: Wed Sep 18 13:38:41 2019
New Revision: 1867116
URL: http://svn.apache.org/viewvc?rev=1867116&view=rev
Log:
OAK-8635 - move LMSEstimator to oak-search, use LMSEstimator in ES
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/LMSEstimator.java
- copied, changed from r1867115,
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimator.java
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/util/LMSEstimatorTest.java
- copied, changed from r1867115,
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimatorTest.java
Removed:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimator.java
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimatorTest.java
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
jackrabbit/oak/trunk/oak-solr-core/pom.xml
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
jackrabbit/oak/trunk/oak-solr-osgi/pom.xml
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java?rev=1867116&r1=1867115&r2=1867116&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java
Wed Sep 18 13:38:41 2019
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugin
import
org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory;
import org.apache.jackrabbit.oak.plugins.index.search.IndexNode;
import org.apache.jackrabbit.oak.plugins.index.search.SizeEstimator;
+import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator;
import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex;
import
org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndexPlanner;
import org.apache.jackrabbit.oak.spi.query.Cursor;
@@ -29,6 +30,8 @@ import org.elasticsearch.common.Strings;
import org.jetbrains.annotations.NotNull;
import java.util.Iterator;
+import java.util.Map;
+import java.util.WeakHashMap;
import java.util.function.Predicate;
import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
@@ -37,6 +40,7 @@ import static org.apache.jackrabbit.oak.
public class ElasticsearchIndex extends FulltextIndex {
private static final Predicate<NodeState>
ELASTICSEARCH_INDEX_DEFINITION_PREDICATE =
state ->
TYPE_ELASTICSEARCH.equals(state.getString(TYPE_PROPERTY_NAME));
+ private static final Map<String, LMSEstimator> estimators = new
WeakHashMap<>();
// higher than some threshold below which the query should rather be
answered by something else if possible
private static final double MIN_COST = 100.1;
@@ -56,10 +60,7 @@ public class ElasticsearchIndex extends
@Override
protected SizeEstimator getSizeEstimator(IndexPlan plan) {
- return () -> {
- // TODO: implement nicely - possible use solr impl's LMSEstimator
- return 2L * (long) MIN_COST;
- };
+ return () ->
getEstimator(plan.getPlanName()).estimate(plan.getFilter());
}
@Override
@@ -101,9 +102,8 @@ public class ElasticsearchIndex extends
final FulltextIndexPlanner.PlanResult pr = getPlanResult(plan);
QueryLimits settings = filter.getQueryLimits();
- Iterator<FulltextResultRow> itr = new
ElasticsearchResultRowIterator(esIndexCoordFactory,
- filter, pr, plan,
- acquireIndexNode(plan), FulltextIndex::shouldInclude);
+ Iterator<FulltextResultRow> itr = new
ElasticsearchResultRowIterator(esIndexCoordFactory, filter, pr, plan,
+ acquireIndexNode(plan), FulltextIndex::shouldInclude,
getEstimator(plan.getPlanName()));
SizeEstimator sizeEstimator = getSizeEstimator(plan);
/*
@@ -115,7 +115,11 @@ public class ElasticsearchIndex extends
// no concept of rewound in ES (even if it might be doing it
internally, we can't do much about it
IteratorRewoundStateProvider rewoundStateProvider = () -> 0;
-
return new FulltextPathCursor(itr, rewoundStateProvider, plan,
settings, sizeEstimator);
}
+
+ private LMSEstimator getEstimator(String path) {
+ estimators.putIfAbsent(path, new LMSEstimator());
+ return estimators.get(path);
+ }
}
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java?rev=1867116&r1=1867115&r2=1867116&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java
Wed Sep 18 13:38:41 2019
@@ -27,6 +27,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator;
import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex;
import
org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndexPlanner.PlanResult;
import org.apache.jackrabbit.oak.spi.query.Filter;
@@ -96,20 +97,22 @@ public class ElasticsearchResultRowItera
private final IndexPlan plan;
private final ElasticsearchIndexNode indexNode;
private final RowInclusionPredicate rowInclusionPredicate;
+ private final LMSEstimator estimator;
ElasticsearchResultRowIterator(@NotNull
ElasticsearchIndexCoordinateFactory esIndexCoordFactory,
@NotNull Filter filter,
@NotNull PlanResult pr,
@NotNull IndexPlan plan,
ElasticsearchIndexNode indexNode,
- RowInclusionPredicate rowInclusionPredicate
- ) {
+ RowInclusionPredicate rowInclusionPredicate,
+ LMSEstimator estimator) {
this.esIndexCoordFactory = esIndexCoordFactory;
this.filter = filter;
this.pr = pr;
this.plan = plan;
this.indexNode = indexNode;
this.rowInclusionPredicate = rowInclusionPredicate != null ?
rowInclusionPredicate : RowInclusionPredicate.NOOP;
+ this.estimator = estimator;
}
@Override
@@ -148,6 +151,8 @@ public class ElasticsearchResultRowItera
SearchHit[] searchHits = docs.getHits().getHits();
PERF_LOGGER.end(start, -1, "{} ...", searchHits.length);
+ estimator.update(filter, docs.getHits().getTotalHits().value);
+
if (searchHits.length < nextBatchSize) {
noDocs = true;
}
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java?rev=1867116&r1=1867115&r2=1867116&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
Wed Sep 18 13:38:41 2019
@@ -374,7 +374,8 @@ public abstract class FulltextIndex impl
private long estimatedSize;
private final int numberOfFacets;
- public FulltextPathCursor(final Iterator<FulltextResultRow> it, final
IteratorRewoundStateProvider iterStateProvider, final IndexPlan plan,
QueryLimits settings, SizeEstimator sizeEstimator) {
+ public FulltextPathCursor(final Iterator<FulltextResultRow> it, final
IteratorRewoundStateProvider iterStateProvider,
+ final IndexPlan plan, QueryLimits settings,
SizeEstimator sizeEstimator) {
pathPrefix = plan.getPathPrefix();
this.sizeEstimator = sizeEstimator;
Iterator<String> pathIterator = new Iterator<String>() {
Copied:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/LMSEstimator.java
(from r1867115,
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimator.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/LMSEstimator.java?p2=jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/LMSEstimator.java&p1=jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimator.java&r1=1867115&r2=1867116&rev=1867116&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimator.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/LMSEstimator.java
Wed Sep 18 13:38:41 2019
@@ -14,45 +14,39 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.plugins.index.solr.query;
+package org.apache.jackrabbit.oak.plugins.index.search.util;
import java.util.Arrays;
import org.apache.jackrabbit.oak.spi.query.Filter;
-import org.apache.solr.common.SolrDocumentList;
/**
* A very simple estimator for no. of entries in the index using least mean
square update method for linear regression.
*/
-class LMSEstimator {
+public class LMSEstimator {
private static final double DEFAULT_ALPHA = 0.03;
- private static final int DEFAULT_THRESHOLD = 5;
+ private static final int DEFAULT_THRESHOLD = 0;
private double[] weights;
private final double alpha;
private final long threshold;
- LMSEstimator(double alpha, double[] weights, long threshold) {
+ private LMSEstimator(double alpha, double[] weights, long threshold) {
this.alpha = alpha;
this.weights = weights;
this.threshold = threshold;
}
- LMSEstimator(double[] weights) {
- this(DEFAULT_ALPHA, weights, DEFAULT_THRESHOLD);
+ public LMSEstimator() {
+ this(DEFAULT_ALPHA, new double[]{0.1,0.2,0.5,0.2,0.1},
DEFAULT_THRESHOLD);
}
- LMSEstimator() {
- this(DEFAULT_ALPHA, new double[5], 5);
- }
-
- synchronized void update(Filter filter, SolrDocumentList docs) {
+ public synchronized void update(Filter filter, long numFound) {
double[] updatedWeights = new double[weights.length];
// least mean square cost
long estimate = estimate(filter);
- long numFound = docs.getNumFound();
long residual = numFound - estimate;
double delta = Math.pow(residual, 2);
@@ -65,7 +59,7 @@ class LMSEstimator {
}
}
- long estimate(Filter filter) {
+ public long estimate(Filter filter) {
long estimatedEntryCount = 0;
for (int i = 0; i < 5; i++) {
estimatedEntryCount += weights[i] * getInput(filter, i);
Copied:
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/util/LMSEstimatorTest.java
(from r1867115,
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimatorTest.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/util/LMSEstimatorTest.java?p2=jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/util/LMSEstimatorTest.java&p1=jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimatorTest.java&r1=1867115&r2=1867116&rev=1867116&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimatorTest.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/util/LMSEstimatorTest.java
Wed Sep 18 13:38:41 2019
@@ -14,12 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.plugins.index.solr.query;
+package org.apache.jackrabbit.oak.plugins.index.search.util;
+import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator;
import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextExpression;
import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextTerm;
import org.apache.jackrabbit.oak.spi.query.Filter;
-import org.apache.solr.common.SolrDocumentList;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -28,7 +28,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
- * Tests for {@link
org.apache.jackrabbit.oak.plugins.index.solr.query.LMSEstimator}
+ * Tests for {@link LMSEstimator}
*/
public class LMSEstimatorTest {
@@ -36,8 +36,8 @@ public class LMSEstimatorTest {
public void testUpdate() throws Exception {
LMSEstimator lmsEstimator = new LMSEstimator();
Filter filter = mock(Filter.class);
- SolrDocumentList docs = mock(SolrDocumentList.class);
- lmsEstimator.update(filter, docs);
+ long numDocs = 100L;
+ lmsEstimator.update(filter, numDocs);
}
@Test
@@ -46,25 +46,23 @@ public class LMSEstimatorTest {
Filter filter = mock(Filter.class);
FullTextExpression fte = new FullTextTerm("foo", "bar", false, false,
"");
when(filter.getFullTextConstraint()).thenReturn(fte);
- SolrDocumentList docs = new SolrDocumentList();
- lmsEstimator.update(filter, docs);
+ lmsEstimator.update(filter, 0);
long actualCount = 10;
- docs.setNumFound(actualCount);
long estimate = lmsEstimator.estimate(filter);
assertEquals(estimate, lmsEstimator.estimate(filter));
long diff = actualCount - estimate;
// update causes weights adjustment
- lmsEstimator.update(filter, docs);
+ lmsEstimator.update(filter, actualCount);
long estimate2 = lmsEstimator.estimate(filter);
assertEquals(estimate2, lmsEstimator.estimate(filter));
long diff2 = actualCount - estimate2;
assertTrue(diff2 < diff); // new estimate is more accurate than
previous one
// update doesn't cause weight adjustments therefore estimates stays
unchanged
- lmsEstimator.update(filter, docs);
+ lmsEstimator.update(filter, actualCount);
long estimate3 = lmsEstimator.estimate(filter);
assertEquals(estimate3, lmsEstimator.estimate(filter));
long diff3 = actualCount - estimate3;
Modified: jackrabbit/oak/trunk/oak-solr-core/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/pom.xml?rev=1867116&r1=1867115&r2=1867116&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-solr-core/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-solr-core/pom.xml Wed Sep 18 13:38:41 2019
@@ -136,6 +136,11 @@
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-search</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-core</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java?rev=1867116&r1=1867115&r2=1867116&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
Wed Sep 18 13:38:41 2019
@@ -17,17 +17,8 @@
package org.apache.jackrabbit.oak.plugins.index.solr.query;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.WeakHashMap;
+import java.util.*;
+
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
@@ -39,6 +30,7 @@ import org.apache.jackrabbit.oak.api.Res
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
import org.apache.jackrabbit.oak.commons.json.JsopWriter;
+import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator;
import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.OakSolrConfiguration;
import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.OakSolrConfigurationProvider;
import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.SolrServerConfigurationProvider;
@@ -232,9 +224,7 @@ public class SolrQueryIndex implements F
}
private synchronized LMSEstimator getEstimator(String path) {
- if (!estimators.containsKey(path)) {
- estimators.put(path, new LMSEstimator());
- }
+ estimators.putIfAbsent(path, new LMSEstimator());
return estimators.get(path);
}
@@ -352,7 +342,7 @@ public class SolrQueryIndex implements F
numFound = docs.getNumFound();
- estimator.update(filter, docs);
+ estimator.update(filter, numFound);
Map<String, Map<String, List<String>>> highlighting =
queryResponse.getHighlighting();
for (SolrDocument doc : docs) {
Modified: jackrabbit/oak/trunk/oak-solr-osgi/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-osgi/pom.xml?rev=1867116&r1=1867115&r2=1867116&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-solr-osgi/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-solr-osgi/pom.xml Wed Sep 18 13:38:41 2019
@@ -112,6 +112,13 @@
<scope>runtime</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-search</artifactId>
+ <version>${project.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+
<!-- Solr -->
<dependency>
<groupId>org.apache.solr</groupId>
