Author: tommaso
Date: Tue Mar 3 09:36:56 2015
New Revision: 1663578
URL: http://svn.apache.org/r1663578
Log:
OAK-2537 - sorting support for solr index, changed estimate count to use a LMS
metric
Added:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimator.java
(with props)
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimatorTest.java
(with props)
Removed:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/AdvancedSolrQueryIndex.java
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/index/SolrIndexEditor.java
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndexProvider.java
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParserTest.java
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrIndexQueryTestIT.java
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/index/SolrIndexEditor.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/index/SolrIndexEditor.java?rev=1663578&r1=1663577&r2=1663578&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/index/SolrIndexEditor.java
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/index/SolrIndexEditor.java
Tue Mar 3 09:36:56 2015
@@ -21,6 +21,9 @@ import java.io.InputStream;
import java.util.LinkedList;
import java.util.List;
+import javax.jcr.PropertyType;
+
+import com.google.common.collect.Iterables;
import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
@@ -31,7 +34,6 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.OakSolrConfiguration;
import org.apache.jackrabbit.oak.spi.commit.Editor;
-import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
@@ -230,18 +232,40 @@ class SolrIndexEditor implements IndexEd
// try to get the field to use for this property from
configuration
String fieldName =
configuration.getFieldNameFor(property.getType());
if (fieldName != null) {
- inputDocument.addField(
- fieldName, property.getValue(property.getType()));
+ Object value = property.getValue(property.getType());
+ inputDocument.addField(fieldName, value);
+ // add sort field
+
inputDocument.addField(getSortingField(property.getType().tag(),
property.getName()), value);
} else {
if (Type.BINARY.tag() == property.getType().tag()) {
- inputDocument.addField(property.getName(),
extractTextValues(property, state));
+ List<String> value = extractTextValues(property,
state);
+ inputDocument.addField(property.getName(), value);
+ StringBuilder builder = new StringBuilder();
+ for (String v : value) {
+ if (builder.length() > 0) {
+ builder.append(',');
+ }
+ builder.append(v);
+ }
+ // add sort field
+
inputDocument.addField(getSortingField(property.getType().tag(),
property.getName()), builder.toString());
} else if (property.isArray()) { // or fallback to adding
propertyName:stringValue(s)
- for (String s : property.getValue(Type.STRINGS)) {
+ Iterable<String> strings =
property.getValue(Type.STRINGS);
+ StringBuilder builder = new StringBuilder();
+ for (String s : strings) {
inputDocument.addField(property.getName(), s);
+ if (builder.length() > 0) {
+ builder.append(',');
+ }
+ builder.append(s);
}
+ // add sort field
+
inputDocument.addField(getSortingField(property.getType().tag(),
property.getName()), builder.toString());
} else {
- inputDocument.addField(
- property.getName(),
property.getValue(Type.STRING));
+ String value = property.getValue(Type.STRING);
+ inputDocument.addField(property.getName(), value);
+ // add sort field
+
inputDocument.addField(getSortingField(property.getType().tag(),
property.getName()), value);
}
}
}
@@ -249,6 +273,21 @@ class SolrIndexEditor implements IndexEd
return inputDocument;
}
+ private String getSortingField(int tag, String s) {
+// switch (tag) {
+// case PropertyType.LONG:
+// return s+"_long_sort";
+// case PropertyType.DATE:
+// return s+"_date_sort";
+// case PropertyType.DOUBLE:
+// return s+"_double_sort";
+// case PropertyType.STRING:
+// return s+"_string_sort";
+// default:
+ return s+"_string_sort";
+// }
+ }
+
private List<String> extractTextValues(
PropertyState property, NodeState state) {
List<String> values = new LinkedList<String>();
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java?rev=1663578&r1=1663577&r2=1663578&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
Tue Mar 3 09:36:56 2015
@@ -17,6 +17,8 @@
package org.apache.jackrabbit.oak.plugins.index.solr.query;
import java.util.Collection;
+import java.util.List;
+import javax.jcr.PropertyType;
import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.OakSolrConfiguration;
import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd;
@@ -26,6 +28,7 @@ import org.apache.jackrabbit.oak.query.f
import org.apache.jackrabbit.oak.query.fulltext.FullTextTerm;
import org.apache.jackrabbit.oak.query.fulltext.FullTextVisitor;
import org.apache.jackrabbit.oak.spi.query.Filter;
+import org.apache.jackrabbit.oak.spi.query.QueryIndex;
import org.apache.solr.client.solrj.SolrQuery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,7 +43,7 @@ class FilterQueryParser {
private static final Logger log =
LoggerFactory.getLogger(FilterQueryParser.class);
- static SolrQuery getQuery(Filter filter, OakSolrConfiguration
configuration) {
+ static SolrQuery getQuery(Filter filter, List<QueryIndex.OrderEntry>
sortOrder, OakSolrConfiguration configuration) {
SolrQuery solrQuery = new SolrQuery();
setDefaults(solrQuery, configuration);
@@ -58,6 +61,24 @@ class FilterQueryParser {
}
}
+ if (sortOrder != null) {
+ for (QueryIndex.OrderEntry orderEntry : sortOrder) {
+ SolrQuery.ORDER order;
+ if
(QueryIndex.OrderEntry.Order.ASCENDING.equals(orderEntry.getOrder())) {
+ order = SolrQuery.ORDER.asc;
+ } else {
+ order = SolrQuery.ORDER.desc;
+ }
+ String sortingField;
+ if ("jcr:path".equals(orderEntry.getPropertyName())) {
+ sortingField = configuration.getPathField();
+ } else {
+ sortingField =
getSortingField(orderEntry.getPropertyType().tag(),
orderEntry.getPropertyName());
+ }
+
solrQuery.addOrUpdateSort(partialEscape(sortingField).toString(), order);
+ }
+ }
+
Collection<Filter.PropertyRestriction> propertyRestrictions =
filter.getPropertyRestrictions();
if (propertyRestrictions != null && !propertyRestrictions.isEmpty()) {
for (Filter.PropertyRestriction pr : propertyRestrictions) {
@@ -218,6 +239,21 @@ class FilterQueryParser {
return solrQuery;
}
+ private static String getSortingField(int tag, String s) {
+// switch (tag) {
+// case PropertyType.LONG:
+// return s+"_long_sort";
+// case PropertyType.DATE:
+// return s+"_date_sort";
+// case PropertyType.DOUBLE:
+// return s+"_double_sort";
+// case PropertyType.STRING:
+// return s+"_string_sort";
+// default:
+ return s+"_string_sort";
+// }
+ }
+
private static CharSequence partialEscape(CharSequence s) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < s.length(); i++) {
Added:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimator.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimator.java?rev=1663578&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimator.java
(added)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimator.java
Tue Mar 3 09:36:56 2015
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.solr.query;
+
+import java.util.Arrays;
+
+import org.apache.jackrabbit.oak.spi.query.Filter;
+import org.apache.solr.common.SolrDocumentList;
+
+/**
+ * A very simple estimator for no. of entries in the index using least mean
square.
+ */
+class LMSEstimator {
+
+ private double[] weights;
+
+ public LMSEstimator(double[] weights) {
+ this.weights = weights;
+ }
+
+ public LMSEstimator() {
+ this.weights = new double[5];
+ }
+
+ synchronized void update(Filter filter, SolrDocumentList docs) {
+ double[] updatedWeights = new double[weights.length];
+ for (int i = 0; i < updatedWeights.length; i++) {
+ double errors = (docs.getNumFound() - estimate(filter)) *
getInput(filter, i);
+ updatedWeights[i] = weights[i] + 0.03 * errors;
+ }
+ // weights updated
+ weights = Arrays.copyOf(updatedWeights, 5);
+ }
+
+ long estimate(Filter filter) {
+ long estimatedEntryCount = 0;
+ for (int i = 0; i < 5; i++) {
+ estimatedEntryCount += weights[i] * getInput(filter, i);
+ }
+ return estimatedEntryCount + 1; // smoothing
+ }
+
+ private long getInput(Filter filter, int i) {
+ assert i < 5;
+ if (i == 0) {
+ return filter.getPropertyRestrictions() != null ?
filter.getPropertyRestrictions().size() : 0;
+ } else if (i == 1) {
+ return filter.containsNativeConstraint() ? 1 : 0;
+ } else if (i == 2) {
+ return filter.getPathRestriction() != null ?
filter.getPathRestriction().ordinal() : 0;
+ } else if (i == 3) {
+ return filter.getPathRestriction() != null ?
filter.getPathRestriction().toString().split("/").length : 0;
+ } else if (i == 4) {
+ return filter.getFullTextConstraint() != null ?
filter.getFullTextConstraint().getPrecedence() : 0;
+ }
+ return 0;
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimator.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java?rev=1663578&r1=1663577&r2=1663578&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
Tue Mar 3 09:36:56 2015
@@ -50,6 +50,7 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SpellCheckResponse;
import org.apache.solr.common.SolrDocument;
@@ -66,7 +67,7 @@ import static org.apache.jackrabbit.oak.
/**
* A Solr based {@link QueryIndex}
*/
-public class SolrQueryIndex implements FulltextQueryIndex {
+public class SolrQueryIndex implements FulltextQueryIndex,
QueryIndex.AdvanceFulltextQueryIndex {
public static final String TYPE = "solr";
@@ -81,16 +82,23 @@ public class SolrQueryIndex implements F
private final OakSolrConfiguration configuration;
private final NodeAggregator aggregator;
+ private final LMSEstimator estimator;
- public SolrQueryIndex(String name, SolrServer solrServer,
OakSolrConfiguration configuration, NodeAggregator aggregator) {
+
+ public SolrQueryIndex(String name, SolrServer solrServer,
OakSolrConfiguration configuration, NodeAggregator aggregator, LMSEstimator
estimator) {
this.name = name;
this.solrServer = solrServer;
this.configuration = configuration;
this.aggregator = aggregator;
+ this.estimator = estimator;
+ }
+
+ public SolrQueryIndex(String name, SolrServer solrServer,
OakSolrConfiguration configuration, NodeAggregator aggregator) {
+ this(name, solrServer, configuration, aggregator, new LMSEstimator());
}
public SolrQueryIndex(String name, SolrServer solrServer,
OakSolrConfiguration configuration) {
- this(name, solrServer, configuration, null);
+ this(name, solrServer, configuration, null, new LMSEstimator());
}
@Override
@@ -150,7 +158,7 @@ public class SolrQueryIndex implements F
@Override
public String getPlan(Filter filter, NodeState nodeState) {
- return FilterQueryParser.getQuery(filter, configuration).toString();
+ return FilterQueryParser.getQuery(filter, null,
configuration).toString();
}
/**
@@ -187,7 +195,11 @@ public class SolrQueryIndex implements F
}
@Override
- public Cursor query(final Filter filter, final NodeState root) {
+ public Cursor query(final IndexPlan plan, final NodeState root) {
+ return query(plan.getFilter(), plan.getSortOrder(), root);
+ }
+
+ private Cursor query(Filter filter, List<OrderEntry> sortOrder, NodeState
root) {
Cursor cursor;
try {
final Set<String> relPaths = filter.getFullTextConstraint() !=
null ? getRelativePaths(filter.getFullTextConstraint())
@@ -196,14 +208,16 @@ public class SolrQueryIndex implements F
final int parentDepth = getDepth(parent);
- cursor = new SolrRowCursor(getIterator(filter, parent,
parentDepth), filter.getQueryEngineSettings());
+ AbstractIterator<SolrResultRow> iterator = getIterator(filter,
sortOrder, parent, parentDepth);
+
+ cursor = new SolrRowCursor(iterator,
filter.getQueryEngineSettings());
} catch (Exception e) {
throw new RuntimeException(e);
}
return cursor;
}
- private AbstractIterator<SolrResultRow> getIterator(final Filter filter,
final String parent, final int parentDepth) {
+ private AbstractIterator<SolrResultRow> getIterator(final Filter filter,
final List<OrderEntry> sortOrder, final String parent, final int parentDepth) {
return new AbstractIterator<SolrResultRow>() {
private final Set<String> seenPaths = Sets.newHashSet();
private final Deque<SolrResultRow> queue = Queues.newArrayDeque();
@@ -258,7 +272,7 @@ public class SolrQueryIndex implements F
if (log.isDebugEnabled()) {
log.debug("converting filter {}", filter);
}
- SolrQuery query = FilterQueryParser.getQuery(filter,
configuration);
+ SolrQuery query = FilterQueryParser.getQuery(filter,
sortOrder, configuration);
if (numFound > 0) {
offset++;
int newOffset = offset * configuration.getRows();
@@ -396,10 +410,6 @@ public class SolrQueryIndex implements F
return fakeDoc;
}
- void onRetrievedDocs(Filter filter, SolrDocumentList docs) {
- // do nothing
- }
-
static boolean isIgnoredProperty(String propertyName, OakSolrConfiguration
configuration) {
return !(NATIVE_LUCENE_QUERY.equals(propertyName) ||
NATIVE_SOLR_QUERY.equals(propertyName)) &&
(!configuration.useForPropertyRestrictions() // Solr index not
used for properties
@@ -409,6 +419,47 @@ public class SolrQueryIndex implements F
||
configuration.getIgnoredProperties().contains(propertyName));
}
+ @Override
+ public List<IndexPlan> getPlans(Filter filter, List<OrderEntry> sortOrder,
NodeState rootState) {
+ // TODO : eventually provide multiple plans for (eventually) filtering
by ACLs
+ // TODO : eventually provide multiple plans for normal paging vs deep
paging
+ if (getMatchingFilterRestrictions(filter) > 0) {
+ return Collections.singletonList(planBuilder(filter)
+ .setEstimatedEntryCount(estimator.estimate(filter))
+ .setSortOrder(sortOrder)
+ .build());
+ } else {
+ return Collections.emptyList();
+ }
+ }
+
+ private IndexPlan.Builder planBuilder(Filter filter) {
+ return new IndexPlan.Builder()
+ .setCostPerExecution(solrServer instanceof EmbeddedSolrServer
? 1 : 2) // disk I/O + network I/O
+ .setCostPerEntry(0.3) // with properly configured SolrCaches
~70% of the doc fetches should hit them
+ .setFilter(filter)
+ .setFulltextIndex(true)
+ .setIncludesNodeData(true) // we currently include node data
+ .setDelayed(true); //Solr is most usually async
+ }
+
+ void onRetrievedDocs(Filter filter, SolrDocumentList docs) {
+ // estimator update
+ estimator.update(filter, docs);
+ }
+
+
+
+ @Override
+ public String getPlanDescription(IndexPlan plan, NodeState root) {
+ return plan.toString();
+ }
+
+ @Override
+ public Cursor query(Filter filter, NodeState rootState) {
+ return query(filter, null, rootState);
+ }
+
static class SolrResultRow {
final String path;
final double score;
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndexProvider.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndexProvider.java?rev=1663578&r1=1663577&r2=1663578&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndexProvider.java
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndexProvider.java
Tue Mar 3 09:36:56 2015
@@ -18,6 +18,8 @@ package org.apache.jackrabbit.oak.plugin
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.WeakHashMap;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -26,15 +28,11 @@ import org.apache.jackrabbit.oak.api.Typ
import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator;
import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.OakSolrConfiguration;
import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.OakSolrConfigurationProvider;
-import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.SolrServerConfiguration;
import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.SolrServerConfigurationProvider;
import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.nodestate.NodeStateSolrServerConfigurationProvider;
-import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.nodestate.NodeStateSolrServerProvider;
import
org.apache.jackrabbit.oak.plugins.index.solr.configuration.nodestate.OakSolrNodeStateConfiguration;
import org.apache.jackrabbit.oak.plugins.index.solr.server.OakSolrServer;
import org.apache.jackrabbit.oak.plugins.index.solr.server.SolrServerProvider;
-import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
-import org.apache.jackrabbit.oak.spi.commit.Observer;
import org.apache.jackrabbit.oak.spi.query.QueryIndex;
import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
@@ -59,6 +57,8 @@ public class SolrQueryIndexProvider impl
private final NodeAggregator aggregator;
+ private final Map<NodeState, LMSEstimator> estimators = new
WeakHashMap<NodeState, LMSEstimator>();
+
public SolrQueryIndexProvider(@Nonnull SolrServerProvider
solrServerProvider, @Nonnull OakSolrConfigurationProvider
oakSolrConfigurationProvider,
@Nullable NodeAggregator nodeAggregator) {
this.oakSolrConfigurationProvider = oakSolrConfigurationProvider;
@@ -87,10 +87,10 @@ public class SolrQueryIndexProvider impl
SolrServerConfigurationProvider
solrServerConfigurationProvider = new
NodeStateSolrServerConfigurationProvider(definition.getChildNode("server"));
SolrServer solrServer = new
OakSolrServer(solrServerConfigurationProvider);
// if it does not already exist I need to register an
observer that updates / closes this SolrServerProvider when the node is
updated/removed
- addQueryIndex(tempIndexes, name, solrServer,
configuration);
+ addQueryIndex(tempIndexes, name, solrServer,
configuration, definition);
} else { // otherwise use the default configuration
providers
OakSolrConfiguration configuration =
oakSolrConfigurationProvider.getConfiguration();
- addQueryIndex(tempIndexes, name,
solrServerProvider.getSearchingSolrServer(), configuration);
+ addQueryIndex(tempIndexes, name,
solrServerProvider.getSearchingSolrServer(), configuration, definition);
}
} catch (Exception e) {
log.warn("could not get Solr query index from node {}",
name, e);
@@ -104,15 +104,23 @@ public class SolrQueryIndexProvider impl
return definition.hasChildNode("server");
}
- private void addQueryIndex(List<QueryIndex> tempIndexes, String name,
SolrServer solrServer, OakSolrConfiguration configuration) {
+ private void addQueryIndex(List<QueryIndex> tempIndexes, String name,
SolrServer solrServer, OakSolrConfiguration configuration, NodeState
definition) {
try {
// the query engine should be returned only if the server is
alive, otherwise other indexes should be used
if (solrServer != null && 0 == solrServer.ping().getStatus()) {
- tempIndexes.add(new AdvancedSolrQueryIndex(
+ LMSEstimator estimator;
+ synchronized (estimators) {
+ estimator = estimators.get(definition);
+ if (estimator == null) {
+ estimator = new LMSEstimator();
+ estimators.put(definition, estimator);
+ }
+ }
+ tempIndexes.add(new SolrQueryIndex(
name,
solrServer,
configuration,
- aggregator));
+ aggregator, estimator));
} else {
if (log.isWarnEnabled()) {
log.warn("cannot create Solr query index as SolrServer {}Â
is unreachable", solrServer);
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml?rev=1663578&r1=1663577&r2=1663578&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
Tue Mar 3 09:36:56 2015
@@ -16,6 +16,7 @@
limitations under the License.
-->
<schema name="minimal" version="1.5">
+
<types>
<fieldType name="string" class="solr.StrField"/>
<fieldType name="descendent_path" class="solr.TextField">
@@ -106,6 +107,7 @@
currencyConfig="currency.xml"/>
<fieldtype name="ignored" stored="false" indexed="false"
multiValued="true" class="solr.StrField"/>
</types>
+
<fields>
<field name="path_exact" type="string" indexed="true" stored="true"/>
<field name="path_child" type="children_path" indexed="true"
stored="false"/>
@@ -118,11 +120,15 @@
<field name=":suggest" type="string" indexed="true" stored="true"
multiValued="true" />
<field name="_version_" type="long" indexed="true" stored="true"/>
- <field name="rep:members" type="string" indexed="true" stored="true"
multiValued="true"/>
- <field name="rep:principalName" type="string" indexed="true"
stored="true" multiValued="true"/>
- <dynamicField name="*_bin" type="string" indexed="true" stored="true"
multiValued="true"/>
+ <!-- sorting dynamic fields -->
+ <!--<dynamicField name="*_long_sort" type="tlong" indexed="false"
stored="false" multiValued="false" docValues="true"/>-->
+ <!--<dynamicField name="*_double_sort" type="tdouble" indexed="false"
stored="false" multiValued="false" docValues="true"/>-->
+ <!--<dynamicField name="*_date_sort" type="tdate" indexed="false"
stored="false" multiValued="false" docValues="true"/>-->
+ <dynamicField name="*_string_sort" type="string" indexed="true"
stored="false" multiValued="false" docValues="true"/>
+
<dynamicField name="*" type="text_general" indexed="true"
stored="true" multiValued="true"/>
</fields>
+
<uniqueKey>path_exact</uniqueKey>
<copyField source="path_exact" dest="path_anc"/>
<copyField source="path_exact" dest="path_des"/>
@@ -131,4 +137,5 @@
<copyField source="*" dest="catch_all"/>
<copyField source="jcr:title" dest=":suggest"/>
<copyField source="jcr:description" dest=":suggest"/>
+
</schema>
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParserTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParserTest.java?rev=1663578&r1=1663577&r2=1663578&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParserTest.java
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParserTest.java
Tue Mar 3 09:36:56 2015
@@ -34,7 +34,7 @@ public class FilterQueryParserTest {
public void testMatchAllConversionWithNoConstraints() throws Exception {
Filter filter = mock(Filter.class);
OakSolrConfiguration configuration = mock(OakSolrConfiguration.class);
- SolrQuery solrQuery = FilterQueryParser.getQuery(filter,
configuration);
+ SolrQuery solrQuery = FilterQueryParser.getQuery(filter, null,
configuration);
assertNotNull(solrQuery);
assertEquals("*:*", solrQuery.getQuery());
}
Added:
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimatorTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimatorTest.java?rev=1663578&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimatorTest.java
(added)
+++
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimatorTest.java
Tue Mar 3 09:36:56 2015
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.solr.query;
+
+import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
+import org.apache.jackrabbit.oak.query.fulltext.FullTextTerm;
+import org.apache.jackrabbit.oak.spi.query.Filter;
+import org.apache.solr.common.SolrDocumentList;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for {@link
org.apache.jackrabbit.oak.plugins.index.solr.query.LMSEstimator}
+ */
+public class LMSEstimatorTest {
+
+ @Test
+ public void testUpdate() throws Exception {
+ LMSEstimator lmsEstimator = new LMSEstimator();
+ Filter filter = mock(Filter.class);
+ SolrDocumentList docs = mock(SolrDocumentList.class);
+ lmsEstimator.update(filter, docs);
+ }
+
+ @Test
+ public void testMultipleUpdatesImproveAccuracy() throws Exception {
+ LMSEstimator lmsEstimator = new LMSEstimator();
+ Filter filter = mock(Filter.class);
+ FullTextExpression fte = new FullTextTerm("foo", "bar", false, false,
"");
+ when(filter.getFullTextConstraint()).thenReturn(fte);
+ SolrDocumentList docs = new SolrDocumentList();
+ lmsEstimator.update(filter, docs);
+ long estimate = lmsEstimator.estimate(filter);
+ docs.setNumFound(10);
+ lmsEstimator.update(filter, docs);
+ long estimate2 = lmsEstimator.estimate(filter);
+ assertTrue(estimate2 > estimate);
+ lmsEstimator.update(filter, docs);
+ long estimate3 = lmsEstimator.estimate(filter);
+ assertTrue(estimate3 > estimate2);
+ }
+
+ @Test
+ public void testEstimate() throws Exception {
+ LMSEstimator lmsEstimator = new LMSEstimator();
+ Filter filter = mock(Filter.class);
+ long estimate = lmsEstimator.estimate(filter);
+ assertEquals(1l, estimate);
+ }
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/LMSEstimatorTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrIndexQueryTestIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrIndexQueryTestIT.java?rev=1663578&r1=1663577&r2=1663578&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrIndexQueryTestIT.java
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrIndexQueryTestIT.java
Tue Mar 3 09:36:56 2015
@@ -446,4 +446,50 @@ public class SolrIndexQueryTestIT extend
assertQuery("//*[jcr:contains(., 'media') and (@p =
'dam/smartcollection' or @p = 'dam/collection') ]", "xpath",
ImmutableList.of(one.getPath(), two.getPath()));
}
+
+ @Test
+ public void testSortingOnPath() throws Exception {
+ Tree test = root.getTree("/").addChild("test");
+ test.addChild("a").addChild("c");
+ test.addChild("b").addChild("d");
+ root.commit();
+
+ Iterator<String> result = executeQuery(
+ "select [jcr:path] from [nt:base] where
isdescendantnode('/test') order by [jcr:path] asc",
+ "JCR-SQL2").iterator();
+ assertTrue(result.hasNext());
+ assertEquals("/test/a", result.next());
+ assertTrue(result.hasNext());
+ assertEquals("/test/a/c", result.next());
+ assertTrue(result.hasNext());
+ assertEquals("/test/b", result.next());
+ assertTrue(result.hasNext());
+ assertEquals("/test/b/d", result.next());
+ assertFalse(result.hasNext());
+ }
+
+ @Test
+ public void testSortingOnProperty() throws Exception {
+ Tree test = root.getTree("/").addChild("test");
+ Tree a = test.addChild("a");
+ a.setProperty("foo", "bar");
+ a.addChild("c").setProperty("foo", "car");
+ Tree b = test.addChild("b");
+ b.setProperty("foo", "tar");
+ b.addChild("d").setProperty("foo", "jar");
+ root.commit();
+
+ Iterator<String> result = executeQuery(
+ "select [jcr:path] from [nt:base] where
isdescendantnode('/test') order by [foo] asc",
+ "JCR-SQL2").iterator();
+ assertTrue(result.hasNext());
+ assertEquals("/test/a", result.next());
+ assertTrue(result.hasNext());
+ assertEquals("/test/a/c", result.next());
+ assertTrue(result.hasNext());
+ assertEquals("/test/b/d", result.next());
+ assertTrue(result.hasNext());
+ assertEquals("/test/b", result.next());
+ assertFalse(result.hasNext());
+ }
}
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml?rev=1663578&r1=1663577&r2=1663578&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
(original)
+++
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
Tue Mar 3 09:36:56 2015
@@ -132,36 +132,12 @@
<field name=":suggest-weight" type="tint" indexed="false"
stored="false" default="1" docValues="true"/>
<field name=":suggest" type="string" indexed="true" stored="true"
multiValued="true" />
- <dynamicField name="*_i" type="int" indexed="true" stored="true"/>
- <dynamicField name="*_is" type="int" indexed="true" stored="true"
multiValued="true"/>
- <dynamicField name="*_s" type="string" indexed="true" stored="true"
/>
- <dynamicField name="*_ss" type="string" indexed="true" stored="true"
multiValued="true"/>
- <dynamicField name="*_l" type="long" indexed="true" stored="true"/>
- <dynamicField name="*_ls" type="long" indexed="true" stored="true"
multiValued="true"/>
- <dynamicField name="*_t" type="text_general" indexed="true"
stored="true"/>
- <dynamicField name="*_txt" type="text_general" indexed="true"
stored="true" multiValued="true"/>
- <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
- <dynamicField name="*_bs" type="boolean" indexed="true" stored="true"
multiValued="true"/>
- <dynamicField name="*_f" type="float" indexed="true" stored="true"/>
- <dynamicField name="*_fs" type="float" indexed="true" stored="true"
multiValued="true"/>
- <dynamicField name="*_d" type="double" indexed="true" stored="true"/>
- <dynamicField name="*_ds" type="double" indexed="true" stored="true"
multiValued="true"/>
+ <!-- sorting dynamic fields -->
+ <!--<dynamicField name="*_long_sort" type="tlong" indexed="false"
stored="false" multiValued="false" docValues="true"/>-->
+ <!--<dynamicField name="*_double_sort" type="tdouble" indexed="false"
stored="false" multiValued="false" docValues="true"/>-->
+ <!--<dynamicField name="*_date_sort" type="tdate" indexed="false"
stored="false" multiValued="false" docValues="true"/>-->
+ <dynamicField name="*_string_sort" type="string" indexed="true"
stored="false" multiValued="false" docValues="true"/>
- <dynamicField name="*_coordinate" type="tdouble" indexed="true"
stored="false" />
-
- <dynamicField name="*_dt" type="date" indexed="true"
stored="true"/>
- <dynamicField name="*_dts" type="date" indexed="true"
stored="true" multiValued="true"/>
- <dynamicField name="*_p" type="location" indexed="true"
stored="true"/>
-
- <dynamicField name="*_ti" type="tint" indexed="true"
stored="true"/>
- <dynamicField name="*_tl" type="tlong" indexed="true"
stored="true"/>
- <dynamicField name="*_tf" type="tfloat" indexed="true"
stored="true"/>
- <dynamicField name="*_td" type="tdouble" indexed="true"
stored="true"/>
- <dynamicField name="*_tdt" type="tdate" indexed="true"
stored="true"/>
-
- <dynamicField name="*_pi" type="pint" indexed="true"
stored="true"/>
- <dynamicField name="*_c" type="currency" indexed="true"
stored="true"/>
- <dynamicField name="*_bin" type="binary" indexed="true"
stored="false" multiValued="true"/>
<dynamicField name="*" type="text_general" indexed="true"
stored="true" multiValued="true"/>
</fields>
<uniqueKey>path_exact</uniqueKey>
