Author: chetanm
Date: Mon Nov 16 07:14:28 2015
New Revision: 1714520
URL: http://svn.apache.org/viewvc?rev=1714520&view=rev
Log:
OAK-3591 - Lucene index with 'analyzed=true' sometimes used by mistake
PropertyDefinition
If index=false then all other config are assumed to be false ignoring whatever
is configured. As if property is not be indexed then it is not considered for
all other cases also
IndexDefinition
Introduced a new `nodeFullTextIndexed` field to treat cases where node is
indexed from cases where only certain property are index with analyzed=true.
This ensures that `areAlMatchingNodeByTypeIndexed` would only return true if
there is a nodeScope indexed field. So far it was returning true even when
there was only a analyzed field
IndexPlanner
For a fulltext query now if the fulltext term is based on node then it only
answers true if and only if node is indexed for fulltext i.e. there at least
one property config with nodeScopeIndex=true
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1714520&r1=1714519&r2=1714520&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
Mon Nov 16 07:14:28 2015
@@ -643,6 +643,7 @@ class IndexDefinition implements Aggrega
final int propertyTypes;
final boolean fulltextEnabled;
final boolean propertyIndexEnabled;
+ final boolean nodeFullTextIndexed;
final Aggregate aggregate;
final Aggregate propAggregate;
@@ -670,8 +671,9 @@ class IndexDefinition implements Aggrega
this.nullCheckEnabledProperties =
ImmutableList.copyOf(nonExistentProperties);
this.notNullCheckEnabledProperties =
ImmutableList.copyOf(existentProperties);
this.fulltextEnabled = aggregate.hasNodeAggregates() ||
hasAnyFullTextEnabledProperty();
+ this.nodeFullTextIndexed = aggregate.hasNodeAggregates() ||
anyNodeScopeIndexedProperty();
this.propertyIndexEnabled = hasAnyPropertyIndexConfigured();
- this.indexesAllNodesOfMatchingType =
allMatchingNodeByTypeIndexed();
+ this.indexesAllNodesOfMatchingType =
areAlMatchingNodeByTypeIndexed();
this.nodeNameIndexed = getOptionalValue(config,
LuceneIndexConstants.INDEX_NODE_NAME, false);
validateRuleDefinition();
}
@@ -698,7 +700,8 @@ class IndexDefinition implements Aggrega
this.nodeScopeAnalyzedProps = original.nodeScopeAnalyzedProps;
this.aggregate = combine(propAggregate, nodeTypeName);
this.fulltextEnabled = aggregate.hasNodeAggregates() ||
original.fulltextEnabled;
- this.indexesAllNodesOfMatchingType =
allMatchingNodeByTypeIndexed();
+ this.nodeFullTextIndexed = aggregate.hasNodeAggregates() ||
original.nodeFullTextIndexed;
+ this.indexesAllNodesOfMatchingType =
areAlMatchingNodeByTypeIndexed();
this.nodeNameIndexed = original.nodeNameIndexed;
}
@@ -787,9 +790,22 @@ class IndexDefinition implements Aggrega
return nodeNameIndexed;
}
+ /**
+ * Returns true if the rule has any property definition which enables
+ * evaluation of fulltext related clauses
+ */
public boolean isFulltextEnabled() {
return fulltextEnabled;
}
+
+ /**
+ * Returns true if a fulltext field for the node would be created
+ * for any node matching the current rule.
+ */
+ public boolean isNodeFullTextIndexed() {
+ return nodeFullTextIndexed;
+ }
+
/**
* @param propertyName name of a property.
* @return the property configuration or <code>null</code> if this
@@ -920,10 +936,26 @@ class IndexDefinition implements Aggrega
return false;
}
- private boolean allMatchingNodeByTypeIndexed(){
- //Incase of fulltext all nodes matching this rule type would be
indexed
- //and would have entry in the index
- if (fulltextEnabled){
+ private boolean anyNodeScopeIndexedProperty(){
+ //Check if there is any nodeScope indexed property as
+ //for such case a node would always be indexed
+ for (PropertyDefinition pd : propConfigs.values()){
+ if (pd.nodeScopeIndex){
+ return true;
+ }
+ }
+
+ for (NamePattern np : namePatterns){
+ if (np.getConfig().nodeScopeIndex){
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private boolean areAlMatchingNodeByTypeIndexed(){
+ if (nodeFullTextIndexed){
return true;
}
@@ -1300,7 +1332,7 @@ class IndexDefinition implements Aggrega
private static boolean hasFulltextEnabledIndexRule(List<IndexingRule>
rules) {
for (IndexingRule rule : rules){
- if (rule.fulltextEnabled){
+ if (rule.isFulltextEnabled()){
return true;
}
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java?rev=1714520&r1=1714519&r2=1714520&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
Mon Nov 16 07:14:28 2015
@@ -266,6 +266,7 @@ class IndexPlanner {
final HashSet<String> relPaths = new HashSet<String>();
final HashSet<String> nonIndexedPaths = new HashSet<String>();
final AtomicBoolean relativeParentsFound = new AtomicBoolean();
+ final AtomicBoolean nodeScopedCondition = new AtomicBoolean();
ft.accept(new FullTextVisitor.FullTextVisitorBase() {
@Override
public boolean visit(FullTextContains contains) {
@@ -308,9 +309,17 @@ class IndexPlanner {
&& !indexingRule.isIndexed(propertyPath)){
nonIndexedPaths.add(p);
}
+
+ if (nodeScopedTerm(propertyName)){
+ nodeScopedCondition.set(true);
+ }
}
});
+ if (nodeScopedCondition.get() &&
!indexingRule.isNodeFullTextIndexed()){
+ return false;
+ }
+
if (relativeParentsFound.get()){
log.debug("Relative parents found {} which are not supported",
relPaths);
return false;
@@ -460,6 +469,14 @@ class IndexPlanner {
return false;
}
+ /**
+ * Determine if the propertyName of a fulltext term indicates current node
+ * @param propertyName property name in the full text term clause
+ */
+ private static boolean nodeScopedTerm(String propertyName) {
+ return propertyName == null || ".".equals(propertyName) ||
"*".equals(propertyName);
+ }
+
//~--------------------------------------------------------< PlanResult >
public static class PlanResult {
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java?rev=1714520&r1=1714519&r2=1714520&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
Mon Nov 16 07:14:28 2015
@@ -360,7 +360,7 @@ public class LuceneIndexEditor implement
//For property index no use making an empty document if
//none of the properties are indexed
- if(!indexingRule.isFulltextEnabled() && !dirty){
+ if(!indexingRule.indexesAllNodesOfMatchingType() && !dirty){
return null;
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java?rev=1714520&r1=1714519&r2=1714520&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java
Mon Nov 16 07:14:28 2015
@@ -103,29 +103,29 @@ class PropertyDefinition {
//By default if a property is defined it is indexed
this.index = getOptionalValue(defn, LuceneIndexConstants.PROP_INDEX,
true);
- this.stored = getOptionalValue(defn,
LuceneIndexConstants.PROP_USE_IN_EXCERPT, false);
- this.nodeScopeIndex = getOptionalValue(defn,
LuceneIndexConstants.PROP_NODE_SCOPE_INDEX, false);
+ this.stored = getOptionalValueIfIndexed(defn,
LuceneIndexConstants.PROP_USE_IN_EXCERPT, false);
+ this.nodeScopeIndex = getOptionalValueIfIndexed(defn,
LuceneIndexConstants.PROP_NODE_SCOPE_INDEX, false);
//If boost is specified then that field MUST be analyzed
if (defn.hasProperty(FIELD_BOOST)){
this.analyzed = true;
} else {
- this.analyzed = getOptionalValue(defn,
LuceneIndexConstants.PROP_ANALYZED, false);
+ this.analyzed = getOptionalValueIfIndexed(defn,
LuceneIndexConstants.PROP_ANALYZED, false);
}
//If node is not set for full text then a property definition
indicates that definition is for property index
- this.propertyIndex = getOptionalValue(defn,
LuceneIndexConstants.PROP_PROPERTY_INDEX, false);
- this.ordered = getOptionalValue(defn,
LuceneIndexConstants.PROP_ORDERED, false);
+ this.propertyIndex = getOptionalValueIfIndexed(defn,
LuceneIndexConstants.PROP_PROPERTY_INDEX, false);
+ this.ordered = getOptionalValueIfIndexed(defn,
LuceneIndexConstants.PROP_ORDERED, false);
this.includedPropertyTypes = IndexDefinition.getSupportedTypes(defn,
LuceneIndexConstants.PROP_INCLUDED_TYPE,
IndexDefinition.TYPES_ALLOW_ALL);
//TODO Add test case for above cases
this.propertyType = getPropertyType(idxDefn, nodeName, defn);
- this.useInSuggest = getOptionalValue(defn,
LuceneIndexConstants.PROP_USE_IN_SUGGEST, false);
- this.useInSpellcheck = getOptionalValue(defn,
LuceneIndexConstants.PROP_USE_IN_SPELLCHECK, false);
- this.nullCheckEnabled = getOptionalValue(defn,
LuceneIndexConstants.PROP_NULL_CHECK_ENABLED, false);
- this.notNullCheckEnabled = getOptionalValue(defn,
LuceneIndexConstants.PROP_NOT_NULL_CHECK_ENABLED, false);
+ this.useInSuggest = getOptionalValueIfIndexed(defn,
LuceneIndexConstants.PROP_USE_IN_SUGGEST, false);
+ this.useInSpellcheck = getOptionalValueIfIndexed(defn,
LuceneIndexConstants.PROP_USE_IN_SPELLCHECK, false);
+ this.nullCheckEnabled = getOptionalValueIfIndexed(defn,
LuceneIndexConstants.PROP_NULL_CHECK_ENABLED, false);
+ this.notNullCheckEnabled = getOptionalValueIfIndexed(defn,
LuceneIndexConstants.PROP_NOT_NULL_CHECK_ENABLED, false);
this.nonRelativeName = determineNonRelativeName();
this.ancestors = computeAncestors(name);
validate();
@@ -201,6 +201,15 @@ class PropertyDefinition {
//~---------------------------------------------< internal >
+ private boolean getOptionalValueIfIndexed(NodeState definition, String
propName, boolean defaultVal){
+ //If property is not to be indexed then all other config would be
+ //set to false ignoring whatever is defined in config for them
+ if (!index){
+ return false;
+ }
+ return getOptionalValue(definition, propName, defaultVal);
+ }
+
private void validate() {
if (nullCheckEnabled && isRegexp){
throw new IllegalStateException(String.format("%s can be set to
true for property definition using " +
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java?rev=1714520&r1=1714519&r2=1714520&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/package-info.java
Mon Nov 16 07:14:28 2015
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-@Version("2.6.0")
+@Version("2.7.0")
@Export(optional = "provide:=true")
package org.apache.jackrabbit.oak.plugins.index.lucene;
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java?rev=1714520&r1=1714519&r2=1714520&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
Mon Nov 16 07:14:28 2015
@@ -682,6 +682,101 @@ public class IndexDefinitionTest {
assertEquals(2, rule1.getNodeScopeAnalyzedProps().size());
}
+ @Test
+ public void nodeFullTextIndexed_Regex() throws Exception {
+ NodeBuilder rules = builder.child(INDEX_RULES);
+ rules.child("nt:folder");
+ TestUtil.child(rules, "nt:folder/properties/prop1")
+ .setProperty(LuceneIndexConstants.PROP_NAME, ".*")
+ .setProperty(LuceneIndexConstants.PROP_ANALYZED, true)
+ .setProperty(LuceneIndexConstants.PROP_IS_REGEX, true);
+
+ IndexDefinition defn = new IndexDefinition(root,
builder.getNodeState());
+ IndexingRule rule =
defn.getApplicableIndexingRule(newTree(newNode("nt:folder")));
+ assertNotNull(rule);
+ assertFalse(rule.isNodeFullTextIndexed());
+
+ TestUtil.child(rules, "nt:folder/properties/prop1")
+ .setProperty(LuceneIndexConstants.PROP_NODE_SCOPE_INDEX, true);
+ defn = new IndexDefinition(root, builder.getNodeState());
+ rule = defn.getApplicableIndexingRule(newTree(newNode("nt:folder")));
+ assertTrue(rule.isNodeFullTextIndexed());
+ assertTrue(rule.indexesAllNodesOfMatchingType());
+ }
+
+ @Test
+ public void nodeFullTextIndexed_Simple() throws Exception {
+ NodeBuilder rules = builder.child(INDEX_RULES);
+ rules.child("nt:folder");
+ TestUtil.child(rules, "nt:folder/properties/prop1")
+ .setProperty(LuceneIndexConstants.PROP_NAME, "foo")
+ .setProperty(LuceneIndexConstants.PROP_ANALYZED, true);
+
+ IndexDefinition defn = new IndexDefinition(root,
builder.getNodeState());
+ IndexingRule rule =
defn.getApplicableIndexingRule(newTree(newNode("nt:folder")));
+ assertNotNull(rule);
+ assertFalse(rule.isNodeFullTextIndexed());
+
+ TestUtil.child(rules, "nt:folder/properties/prop1")
+ .setProperty(LuceneIndexConstants.PROP_NODE_SCOPE_INDEX, true);
+ defn = new IndexDefinition(root, builder.getNodeState());
+ rule = defn.getApplicableIndexingRule(newTree(newNode("nt:folder")));
+ assertTrue(rule.isNodeFullTextIndexed());
+ assertTrue(rule.indexesAllNodesOfMatchingType());
+ }
+
+ @Test
+ public void nodeFullTextIndexed_Aggregates() throws Exception {
+ NodeBuilder rules = builder.child(INDEX_RULES);
+ rules.child("nt:folder");
+ TestUtil.child(rules, "nt:folder/properties/prop1")
+ .setProperty(LuceneIndexConstants.PROP_NAME, "foo")
+ .setProperty(LuceneIndexConstants.PROP_ANALYZED, true);
+
+ NodeBuilder aggregates =
builder.child(LuceneIndexConstants.AGGREGATES);
+ NodeBuilder aggFolder = aggregates.child("nt:folder");
+ aggFolder.child("i1").setProperty(LuceneIndexConstants.AGG_PATH, "*");
+
+ IndexDefinition defn = new IndexDefinition(root,
builder.getNodeState());
+ IndexingRule rule =
defn.getApplicableIndexingRule(newTree(newNode("nt:folder")));
+ assertNotNull(rule);
+ assertTrue(rule.isNodeFullTextIndexed());
+ assertTrue(rule.indexesAllNodesOfMatchingType());
+ }
+
+ @Test
+ public void nonIndexPropShouldHaveAllOtherConfigDisabled() throws
Exception{
+ NodeBuilder rules = builder.child(INDEX_RULES);
+ rules.child("nt:folder");
+ TestUtil.child(rules, "nt:folder/properties/prop1")
+ .setProperty(LuceneIndexConstants.PROP_NAME, "foo")
+ .setProperty(LuceneIndexConstants.PROP_INDEX, false)
+ .setProperty(LuceneIndexConstants.PROP_USE_IN_SUGGEST, true)
+ .setProperty(LuceneIndexConstants.PROP_USE_IN_SPELLCHECK, true)
+ .setProperty(LuceneIndexConstants.PROP_NULL_CHECK_ENABLED,
true)
+ .setProperty(LuceneIndexConstants.PROP_NOT_NULL_CHECK_ENABLED,
true)
+ .setProperty(LuceneIndexConstants.PROP_USE_IN_EXCERPT, true)
+ .setProperty(LuceneIndexConstants.PROP_NODE_SCOPE_INDEX, true)
+ .setProperty(LuceneIndexConstants.PROP_ORDERED, true)
+ .setProperty(LuceneIndexConstants.PROP_ANALYZED, true);
+ IndexDefinition defn = new IndexDefinition(root,
builder.getNodeState());
+ IndexingRule rule =
defn.getApplicableIndexingRule(newTree(newNode("nt:folder")));
+ assertNotNull(rule);
+
+ PropertyDefinition pd = rule.getConfig("foo");
+ //Assert that all other config is false if the index=false for any
property
+ assertFalse(pd.index);
+ assertFalse(pd.nodeScopeIndex);
+ assertFalse(pd.useInSuggest);
+ assertFalse(pd.useInSpellcheck);
+ assertFalse(pd.nullCheckEnabled);
+ assertFalse(pd.notNullCheckEnabled);
+ assertFalse(pd.stored);
+ assertFalse(pd.ordered);
+ assertFalse(pd.analyzed);
+
+ }
+
//TODO indexesAllNodesOfMatchingType - with nullCheckEnabled
private static IndexingRule getRule(IndexDefinition defn, String typeName){
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java?rev=1714520&r1=1714519&r2=1714520&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java
Mon Nov 16 07:14:28 2015
@@ -422,7 +422,6 @@ public class IndexPlannerTest {
assertNotNull(planner.getPlan());
}
- @Ignore("OAK-3591")
@Test
public void noPlanForFulltextQueryAndOnlyAnalyzedProperties() throws
Exception{
NodeBuilder defn = newLucenePropertyIndexDefinition(builder, "test",
of("foo"), "async");
@@ -441,7 +440,6 @@ public class IndexPlannerTest {
assertNull(plan);
}
- @Ignore("OAK-3591")
@Test
public void noPlanForNodeTypeQueryAndOnlyAnalyzedProperties() throws
Exception{
NodeBuilder defn = newLucenePropertyIndexDefinition(builder, "test",
of("foo"), "async");
