Author: tommaso
Date: Thu Apr 12 11:46:17 2018
New Revision: 1828972

URL: http://svn.apache.org/viewvc?rev=1828972&view=rev
Log:
OAK-3336 - copying first lucene agnostic utils from oak-lucene into oak-search

Added:
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/Aggregate.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/BadIndexTracker.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexFormatVersion.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexLookup.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexNode.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/NodeStateCloner.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/PropertyDefinition.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/PropertyUpdateCallback.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/SizeEstimator.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/ConfigUtil.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/FunctionIndexProcessor.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/IndexDefinitionBuilder.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/IndexHelper.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/NodeStateCopyUtils.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/
    
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/AggregateTest.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinitionTest.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/TestUtil.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/util/
    
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/util/FunctionIndexProcessorTest.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/util/IndexDefinitionBuilderTest.java
   (with props)
    
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/util/NodeStateCopyUtilsTest.java
   (with props)
Modified:
    jackrabbit/oak/trunk/oak-search/pom.xml

Modified: jackrabbit/oak/trunk/oak-search/pom.xml
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/pom.xml?rev=1828972&r1=1828971&r2=1828972&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-search/pom.xml Thu Apr 12 11:46:17 2018
@@ -54,7 +54,7 @@
                         <phase>pre-integration-test</phase>
                         <configuration>
                             <!--
-                              This is required because there's no prior 
(stable) version of oak-search-mt.
+                              This is required because there's no prior 
(stable) version of oak-search.
                               This should be removed post 1.8 release
                               Anyway nothing is exported therefore it 
shouldn't be a problem
                             -->
@@ -94,6 +94,24 @@
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.jackrabbit</groupId>
+            <artifactId>oak-core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.jackrabbit</groupId>
+            <artifactId>oak-store-spi</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.jackrabbit</groupId>
+            <artifactId>oak-query-spi</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
 
         <!-- Logging -->
         <dependency>
@@ -121,6 +139,18 @@
             <version>1.10.19</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-all</artifactId>
+            <version>1.3</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.jackrabbit</groupId>
+            <artifactId>oak-jcr</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
 
     </dependencies>
 </project>

Added: 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/Aggregate.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/Aggregate.java?rev=1828972&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/Aggregate.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/Aggregate.java
 Thu Apr 12 11:46:17 2018
@@ -0,0 +1,660 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import javax.annotation.CheckForNull;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.index.search.util.ConfigUtil;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.Iterables.toArray;
+import static com.google.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Lists.newArrayListWithCapacity;
+import static org.apache.jackrabbit.oak.commons.PathUtils.elements;
+import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath;
+
+class Aggregate {
+
+    public static final String MATCH_ALL = "*";
+
+    /**
+     * recursive aggregation (for same type nodes) limit default value.
+     */
+    public static final int RECURSIVE_AGGREGATION_LIMIT_DEFAULT = 5;
+    private final String nodeTypeName;
+    private final List<? extends Include> includes;
+    final int reAggregationLimit;
+    private final List<NodeInclude> relativeNodeIncludes;
+    private final boolean nodeAggregates;
+
+    Aggregate(String nodeTypeName) {
+       this(nodeTypeName, Collections.<Include>emptyList());
+    }
+
+    Aggregate(String nodeTypeName, List<? extends Include> includes) {
+        this(nodeTypeName, includes, RECURSIVE_AGGREGATION_LIMIT_DEFAULT);
+    }
+
+    Aggregate(String nodeTypeName, List<? extends Include> includes,
+              int recursionLimit) {
+        this.nodeTypeName = nodeTypeName;
+        this.includes = ImmutableList.copyOf(includes);
+        this.reAggregationLimit = recursionLimit;
+        this.relativeNodeIncludes = findRelativeNodeIncludes(includes);
+        this.nodeAggregates = hasNodeIncludes(includes);
+    }
+
+    public List<? extends Include> getIncludes() {
+        return includes;
+    }
+
+    public void collectAggregates(NodeState root, ResultCollector collector) {
+        if (matchingType(nodeTypeName, root)) {
+            List<Matcher> matchers = createMatchers();
+            collectAggregates(root, matchers, collector);
+        }
+    }
+
+    public List<Matcher> createMatchers(AggregateRoot root){
+        List<Matcher> matchers = newArrayListWithCapacity(includes.size());
+        for (Include include : includes) {
+            matchers.add(new Matcher(this, include, root));
+        }
+        return matchers;
+    }
+
+    public boolean hasRelativeNodeInclude(String nodePath) {
+        for (NodeInclude ni : relativeNodeIncludes){
+            if (ni.matches(nodePath)){
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public boolean hasNodeAggregates(){
+        return nodeAggregates;
+    }
+
+    @Override
+    public String toString() {
+        return nodeTypeName;
+    }
+
+    private static boolean matchingType(String nodeTypeName, NodeState 
nodeState) {
+        if (nodeTypeName.equals(ConfigUtil.getPrimaryTypeName(nodeState))) {
+            return true;
+        }
+
+        for (String mixin : ConfigUtil.getMixinNames(nodeState)) {
+            if (nodeTypeName.equals(mixin)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private static void collectAggregates(NodeState nodeState, List<Matcher> 
matchers,
+                                          ResultCollector collector) {
+        if (hasPatternMatcher(matchers)){
+            collectAggregatesForPatternMatchers(nodeState, matchers, 
collector);
+        } else {
+            collectAggregatesForDirectMatchers(nodeState, matchers, collector);
+        }
+    }
+
+    private static void collectAggregatesForDirectMatchers(NodeState 
nodeState, List<Matcher> matchers,
+                                          ResultCollector collector) {
+        Map<String, ChildNodeEntry> children = Maps.newHashMap();
+        //Collect potentially matching child nodestates based on matcher name
+        for (Matcher m : matchers){
+            String nodeName = m.getNodeName();
+            NodeState child = nodeState.getChildNode(nodeName);
+            if (child.exists()){
+                children.put(nodeName, new MemoryChildNodeEntry(nodeName, 
child));
+            }
+        }
+        matchChildren(matchers, collector, children.values());
+    }
+
+    private static void collectAggregatesForPatternMatchers(NodeState 
nodeState, List<Matcher> matchers,
+                                          ResultCollector collector) {
+        matchChildren(matchers, collector, nodeState.getChildNodeEntries());
+    }
+
+    private static void matchChildren(List<Matcher> matchers, ResultCollector 
collector,
+                                      Iterable<? extends ChildNodeEntry> 
children) {
+        for (ChildNodeEntry cne : children) {
+            List<Matcher> nextSet = newArrayListWithCapacity(matchers.size());
+            for (Matcher m : matchers) {
+                Matcher result = m.match(cne.getName(), cne.getNodeState());
+                if (result.getStatus() == Matcher.Status.MATCH_FOUND){
+                    result.collectResults(collector);
+                }
+
+                if (result.getStatus() != Matcher.Status.FAIL){
+                    nextSet.addAll(result.nextSet());
+                }
+            }
+            if (!nextSet.isEmpty()) {
+                collectAggregates(cne.getNodeState(), nextSet, collector);
+            }
+        }
+    }
+
+    private static boolean hasPatternMatcher(List<Matcher> matchers){
+        for (Matcher m : matchers){
+            if (m.isPatternBased()){
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private List<Matcher> createMatchers() {
+        List<Matcher> matchers = newArrayListWithCapacity(includes.size());
+        for (Include include : includes) {
+            matchers.add(new Matcher(this, include));
+        }
+        return matchers;
+    }
+
+    private static List<NodeInclude> findRelativeNodeIncludes(List<? extends 
Include> includes) {
+        List<NodeInclude> result = newArrayList();
+        for (Include i : includes){
+            if (i instanceof NodeInclude){
+                NodeInclude ni = (NodeInclude) i;
+                if (ni.relativeNode){
+                    result.add(ni);
+                }
+            }
+        }
+        return ImmutableList.copyOf(result);
+    }
+
+    private static boolean hasNodeIncludes(List<? extends Include> includes) {
+        return Iterables.any(includes, new Predicate<Include>() {
+            @Override
+            public boolean apply(Include input) {
+                return input instanceof NodeInclude;
+            }
+        });
+    }
+
+    public static interface AggregateMapper {
+        @CheckForNull
+        Aggregate getAggregate(String nodeTypeName);
+    }
+
+    //~-----------------------------------------------------< Includes >
+
+    public static abstract class Include<T> {
+        protected final String[] elements;
+
+        public Include(String pattern) {
+            this.elements = computeElements(pattern);
+        }
+
+        public boolean match(String name, NodeState nodeState, int depth) {
+            String element = elements[depth];
+            if (MATCH_ALL.equals(element)) {
+                return true;
+            } else if (element.equals(name)) {
+                return true;
+            }
+            return false;
+        }
+
+        public int maxDepth() {
+            return elements.length;
+        }
+
+        public void collectResults(T rootInclude, String rootIncludePath,
+                                   String nodePath, NodeState nodeState,  
ResultCollector results) {
+            collectResults(nodePath, nodeState, results);
+        }
+
+        public void collectResults(String nodePath, NodeState nodeState,
+                                            ResultCollector results) {
+
+        }
+
+        public abstract boolean aggregatesProperty(String name);
+
+        @CheckForNull
+        public Aggregate getAggregate(NodeState matchedNodeState) {
+            return null;
+        }
+
+        public boolean isPattern(int depth){
+            return MATCH_ALL.equals(elements[depth]);
+
+        }
+
+        public String getElementNameIfNotAPattern(int depth) {
+            checkArgument(!isPattern(depth),
+                    "Element at %s is pattern instead of specific name in %s", 
depth, Arrays.toString(elements));
+            return elements[depth];
+        }
+    }
+
+    public static class NodeInclude extends Include<NodeInclude> {
+        final String primaryType;
+        final boolean relativeNode;
+        private final String pattern;
+        private final AggregateMapper aggMapper;
+
+        public NodeInclude(AggregateMapper mapper, String pattern) {
+            this(mapper, null, pattern, false);
+        }
+
+        public NodeInclude(AggregateMapper mapper, String primaryType, String 
pattern, boolean relativeNode) {
+            super(pattern);
+            this.pattern = pattern;
+            this.primaryType = primaryType;
+            this.aggMapper = mapper;
+            this.relativeNode = relativeNode;
+        }
+
+        @Override
+        public boolean match(String name, NodeState nodeState, int depth) {
+            //As per JR2 the primaryType is enforced on last element
+            //last segment -> add to collector if node type matches
+            if (depth == maxDepth() - 1
+                    && primaryType != null
+                    && !matchingType(primaryType, nodeState)) {
+                return false;
+            }
+            return super.match(name, nodeState, depth);
+        }
+
+        @Override
+        public void collectResults(NodeInclude rootInclude, String 
rootIncludePath, String nodePath,
+                                   NodeState nodeState, ResultCollector 
results) {
+            //For supporting jcr:contains(jcr:content, 'foo')
+            if (rootInclude.relativeNode){
+                results.onResult(new NodeIncludeResult(nodePath, 
rootIncludePath, nodeState));
+            }
+
+            //For supporting jcr:contains(., 'foo')
+            results.onResult(new NodeIncludeResult(nodePath, nodeState));
+        }
+
+        @Override
+        public boolean aggregatesProperty(String name) {
+            return true;
+        }
+
+        @Override
+        public Aggregate getAggregate(NodeState matchedNodeState) {
+            //Check agg defn for primaryType first
+            Aggregate agg = 
aggMapper.getAggregate(ConfigUtil.getPrimaryTypeName(matchedNodeState));
+
+            //If not found then look for defn for mixins
+            if (agg == null) {
+                for (String mixin : 
ConfigUtil.getMixinNames(matchedNodeState)) {
+                    agg = aggMapper.getAggregate(mixin);
+                    if (agg != null) {
+                        break;
+                    }
+                }
+            }
+            return agg;
+        }
+
+        @Override
+        public String toString() {
+            return "NodeInclude{" +
+                    "primaryType='" + primaryType + '\'' +
+                    ", relativeNode=" + relativeNode +
+                    ", pattern='" + pattern + '\'' +
+                    '}';
+        }
+
+        public boolean matches(String nodePath) {
+            List<String> pathElements = 
ImmutableList.copyOf(PathUtils.elements(nodePath));
+            if (pathElements.size() != elements.length){
+                return false;
+            }
+
+            for (int i = 0; i < elements.length; i++){
+                String element = elements[i];
+                if (MATCH_ALL.equals(element)) {
+                    continue;
+                }
+
+                if (!element.equals(pathElements.get(i))) {
+                    return false;
+                }
+            }
+            return true;
+        }
+    }
+
+    public static class PropertyInclude extends Include<PropertyInclude> {
+        private final PropertyDefinition propertyDefinition;
+        private final String propertyName;
+        private final Pattern pattern;
+        private final String parentPath;
+
+        public PropertyInclude(PropertyDefinition pd) {
+            super(getParentPath(pd.name));
+            this.propertyDefinition = pd;
+            this.propertyName = PathUtils.getName(pd.name);
+            this.parentPath = getParentPath(pd.name);
+
+            if (pd.isRegexp) {
+                pattern = Pattern.compile(propertyName);
+            } else {
+                pattern = null;
+            }
+        }
+
+        @Override
+        public void collectResults(String nodePath, NodeState nodeState, 
ResultCollector results) {
+            if (pattern != null) {
+                for (PropertyState ps : nodeState.getProperties()) {
+                    if (pattern.matcher(ps.getName()).matches()) {
+                        results.onResult(new PropertyIncludeResult(ps, 
propertyDefinition, parentPath));
+                    }
+                }
+            } else {
+                PropertyState ps = nodeState.getProperty(propertyName);
+                if (ps != null) {
+                    results.onResult(new PropertyIncludeResult(ps, 
propertyDefinition, parentPath));
+                }
+            }
+        }
+
+        @Override
+        public boolean aggregatesProperty(String name) {
+            if (pattern != null){
+                return pattern.matcher(name).matches();
+            }
+            return propertyName.equals(name);
+        }
+
+        @Override
+        public String toString() {
+            return propertyDefinition.toString();
+        }
+
+        public PropertyDefinition getPropertyDefinition() {
+            return propertyDefinition;
+        }
+    }
+
+    public static interface ResultCollector {
+        void onResult(NodeIncludeResult result);
+
+        void onResult(PropertyIncludeResult result);
+    }
+
+    public static class NodeIncludeResult {
+        final NodeState nodeState;
+        final String nodePath;
+        final String rootIncludePath;
+
+        public NodeIncludeResult(String nodePath, NodeState nodeState) {
+            this(nodePath, null, nodeState);
+        }
+
+        public NodeIncludeResult(String nodePath, String rootIncludePath, 
NodeState nodeState) {
+            this.nodePath = nodePath;
+            this.nodeState = nodeState;
+            this.rootIncludePath = rootIncludePath;
+        }
+
+        public boolean isRelativeNode(){
+            return  rootIncludePath != null;
+        }
+
+        @Override
+        public String toString() {
+            return "NodeIncludeResult{" +
+                    "nodePath='" + nodePath + '\'' +
+                    ", rootIncludePath='" + rootIncludePath + '\'' +
+                    '}';
+        }
+    }
+
+    public static class PropertyIncludeResult {
+        final PropertyState propertyState;
+        final PropertyDefinition pd;
+        final String propertyPath;
+        final String nodePath;
+
+        public PropertyIncludeResult(PropertyState propertyState, 
PropertyDefinition pd,
+                                     String parentPath) {
+            this.propertyState = propertyState;
+            this.pd = pd;
+            this.nodePath = parentPath;
+            this.propertyPath = PathUtils.concat(parentPath, 
propertyState.getName());
+        }
+    }
+
+    public interface AggregateRoot {
+        void markDirty();
+
+        String getPath();
+    }
+
+    public static class Matcher {
+        public enum Status {CONTINUE, MATCH_FOUND, FAIL}
+
+        private static class RootState {
+            final AggregateRoot root;
+            final Aggregate rootAggregate;
+            final Include rootInclude;
+
+            private RootState(AggregateRoot root, Aggregate rootAggregate, 
Include rootInclude) {
+                this.root = root;
+                this.rootAggregate = rootAggregate;
+                this.rootInclude = rootInclude;
+            }
+        }
+
+        private final RootState rootState;
+        private final Include currentInclude;
+        /**
+         * Current depth in the include pattern.
+         */
+        private final int depth;
+        private final Status status;
+        private final NodeState matchedNodeState;
+        private final String currentPath;
+
+        private final List<String> aggregateStack;
+
+        public Matcher(Aggregate aggregate, Include currentInclude) {
+            this(aggregate, currentInclude, null);
+        }
+
+        public Matcher(Aggregate aggregate, Include include, AggregateRoot 
root) {
+            this.rootState = new RootState(root, aggregate, include);
+            this.depth = 0;
+            this.currentInclude = include;
+            this.status = Status.CONTINUE;
+            this.currentPath = null;
+            this.matchedNodeState = null;
+            this.aggregateStack = Collections.emptyList();
+        }
+
+        private Matcher(Matcher m, Status status, int depth) {
+            checkArgument(status == Status.FAIL);
+            this.rootState = m.rootState;
+            this.depth = depth;
+            this.currentInclude = m.currentInclude;
+            this.status = status;
+            this.currentPath = null;
+            this.matchedNodeState = null;
+            this.aggregateStack = m.aggregateStack;
+        }
+
+        private Matcher(Matcher m, Status status, int depth,
+                        NodeState matchedNodeState, String currentPath) {
+            checkArgument(status != Status.FAIL);
+            this.rootState = m.rootState;
+            this.depth = depth;
+            this.currentInclude = m.currentInclude;
+            this.status = status;
+            this.matchedNodeState = matchedNodeState;
+            this.currentPath = currentPath;
+            this.aggregateStack = m.aggregateStack;
+        }
+
+        private Matcher(Matcher m, Include i, String currentPath) {
+            checkArgument(m.status == Status.MATCH_FOUND);
+            this.rootState = m.rootState;
+            this.depth = 0;
+            this.currentInclude = i;
+            this.status = Status.CONTINUE;
+            this.matchedNodeState = null;
+            this.currentPath = currentPath;
+
+            List<String> paths = newArrayList(m.aggregateStack);
+            paths.add(currentPath);
+            this.aggregateStack = ImmutableList.copyOf(paths);
+        }
+
+        public boolean isPatternBased() {
+            return currentInclude.isPattern(depth);
+        }
+
+        /**
+         * Returns the nodeName at current depth. This should only be called
+         * if and only if #isPatternBased is false otherwise it would throw 
exception
+         */
+        public String getNodeName() {
+            return currentInclude.getElementNameIfNotAPattern(depth);
+        }
+
+        public Matcher match(String name, NodeState nodeState) {
+            boolean result = currentInclude.match(name, nodeState, depth);
+            if (result){
+                if (hasMore()){
+                    return new Matcher(this, Status.CONTINUE, depth, 
nodeState, path(name));
+                } else {
+                    return new Matcher(this, Status.MATCH_FOUND, depth, 
nodeState, path(name));
+                }
+            } else {
+                return new Matcher(this, Status.FAIL, depth);
+            }
+        }
+
+        public Collection<Matcher> nextSet() {
+            checkArgument(status != Status.FAIL);
+
+            if (status == Status.MATCH_FOUND){
+                Aggregate nextAgg = 
currentInclude.getAggregate(matchedNodeState);
+                if (nextAgg != null){
+                    int recursionLevel = aggregateStack.size() + 1;
+
+                    if (recursionLevel >= 
rootState.rootAggregate.reAggregationLimit){
+                        return Collections.emptyList();
+                    }
+
+                    List<Matcher> result = 
Lists.newArrayListWithCapacity(nextAgg.includes.size());
+                    for (Include i : nextAgg.includes){
+                        result.add(new Matcher(this,  i, currentPath));
+                    }
+                    return result;
+                }
+                return Collections.emptyList();
+            }
+
+            return Collections.singleton(new Matcher(this, status, depth + 1,
+                    null, currentPath));
+        }
+
+        public void collectResults(ResultCollector results) {
+            checkArgument(status == Status.MATCH_FOUND);
+
+            //If result being collected as part of reaggregation then take path
+            //from the stack otherwise its the current path
+            String rootIncludePath = aggregateStack.isEmpty() ?  currentPath : 
aggregateStack.get(0);
+            currentInclude.collectResults(rootState.rootInclude, 
rootIncludePath,
+                    currentPath, matchedNodeState, results);
+        }
+
+        public void markRootDirty() {
+            checkArgument(status == Status.MATCH_FOUND);
+            rootState.root.markDirty();
+        }
+
+        public String getRootPath() {
+            return rootState.root.getPath();
+        }
+
+        public String getMatchedPath(){
+            checkArgument(status == Status.MATCH_FOUND);
+            return currentPath;
+        }
+
+        public Include getCurrentInclude(){
+            return currentInclude;
+        }
+
+        public Status getStatus() {
+            return status;
+        }
+
+        public boolean aggregatesProperty(String name) {
+            checkArgument(status == Status.MATCH_FOUND);
+            return currentInclude.aggregatesProperty(name);
+        }
+
+        private boolean hasMore() {
+            return depth < currentInclude.maxDepth() - 1;
+        }
+
+        private String path(String nodeName){
+            if (currentPath == null){
+                return nodeName;
+            } else {
+                return PathUtils.concat(currentPath, nodeName);
+            }
+        }
+    }
+
+    //~--------------------------------------------------< utility >
+
+    private static String[] computeElements(String path) {
+        return toArray(elements(path), String.class);
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/Aggregate.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/BadIndexTracker.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/BadIndexTracker.java?rev=1828972&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/BadIndexTracker.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/BadIndexTracker.java
 Thu Apr 12 11:46:17 2018
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import com.google.common.base.Ticker;
+import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class BadIndexTracker {
+    /**
+     * Time interval in millis after which a bad index would be accessed again
+     * to check if it has been fixed
+     */
+    private static final long DEFAULT_RECHECK_INTERVAL = 
TimeUnit.MINUTES.toMillis(15);
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
+    private final Map<String, BadIndexInfo> badIndexesForRead = 
Maps.newConcurrentMap();
+    private final Map<String, BadIndexInfo> badPersistedIndexes = 
Maps.newConcurrentMap();
+    private final long recheckIntervalMillis;
+    private Ticker ticker = Ticker.systemTicker();
+    private int indexerCycleCount;
+
+    public BadIndexTracker() {
+        this(DEFAULT_RECHECK_INTERVAL);
+    }
+
+    public BadIndexTracker(long recheckIntervalMillis) {
+        this.recheckIntervalMillis = recheckIntervalMillis;
+        log.info("Bad Index recheck interval set to {} seconds",
+                TimeUnit.MILLISECONDS.toSeconds(recheckIntervalMillis));
+    }
+
+    public void markGoodIndexes(Set<String> updatedIndexPaths) {
+        indexerCycleCount++;
+        for (String indexPath : updatedIndexPaths) {
+            markGoodIndex(indexPath);
+        }
+    }
+
+    public void markGoodIndex(String indexPath) {
+        BadIndexInfo info = badIndexesForRead.remove(indexPath);
+        badPersistedIndexes.remove(indexPath);
+        if (info != null) {
+            log.info("Index [{}] which was not working {} is found to be 
healthy again",
+                    indexPath, info.getStats());
+        }
+    }
+
+    /**
+     * Invoked to mark a persisted index as bad i.e. where exception is thrown 
when index is reopened
+     * after update
+     *
+     * @param path index path
+     * @param e exception
+     */
+    public void markBadPersistedIndex(String path, Throwable e) {
+        BadIndexInfo badIndex = badPersistedIndexes.get(path);
+        if (badIndex == null) {
+            badPersistedIndexes.put(path, new BadIndexInfo(path, e, true));
+            log.error("Could not open the Lucene index at [{}]", path, e);
+        } else {
+            badIndex.failedAccess(e);
+            log.error("Could not open the Lucene index at [{}] . {}",
+                    path, badIndex.getStats(), e);
+        }
+    }
+
+    /**
+     * Invoked to mark a local index as bad i.e. where exception was thrown 
when index was
+     * opened for query. It can h
+     */
+    public void markBadIndexForRead(String path, Throwable e) {
+        BadIndexInfo badIndex = badIndexesForRead.get(path);
+        if (badIndex == null) {
+            badIndexesForRead.put(path, new BadIndexInfo(path, e, false));
+            log.error("Could not access the Lucene index at [{}]", path, e);
+        } else {
+            badIndex.failedAccess(e);
+            log.error("Could not access the Lucene index at [{}] . {}",
+                    path, badIndex.getStats(), e);
+        }
+    }
+
+    public boolean isIgnoredBadIndex(String path) {
+        BadIndexInfo badIdx = badIndexesForRead.get(path);
+        if (badIdx == null) {
+            return false;
+        }
+        return !badIdx.tryAgain();
+    }
+
+    public Set<String> getIndexPaths() {
+        return badIndexesForRead.keySet();
+    }
+
+    BadIndexInfo getInfo(String indexPath){
+        return badIndexesForRead.get(indexPath);
+    }
+
+    Set<String> getBadPersistedIndexPaths() {
+        return badPersistedIndexes.keySet();
+    }
+
+    BadIndexInfo getPersistedIndexInfo(String indexPath){
+        return badPersistedIndexes.get(indexPath);
+    }
+
+    public long getRecheckIntervalMillis() {
+        return recheckIntervalMillis;
+    }
+
+    void setTicker(Ticker ticker) {
+        this.ticker = ticker;
+    }
+
+    public boolean hasBadIndexes(){
+        return !(badIndexesForRead.isEmpty() && badPersistedIndexes.isEmpty());
+    }
+
+    class BadIndexInfo {
+        final String path;
+        final int lastIndexerCycleCount = indexerCycleCount;
+        private final long createdTime = 
TimeUnit.NANOSECONDS.toMillis(ticker.read());
+        private final boolean persistedIndex;
+        private final Stopwatch created = Stopwatch.createStarted(ticker);
+        private final Stopwatch watch = Stopwatch.createStarted(ticker);
+        private String exception;
+        private int accessCount;
+        private int failedAccessCount;
+
+
+        public BadIndexInfo(String path, Throwable e, boolean persistedIndex) {
+            this.path = path;
+            this.exception = Throwables.getStackTraceAsString(e);
+            this.persistedIndex = persistedIndex;
+        }
+
+        public boolean tryAgain() {
+            accessCount++;
+
+            if (watch.elapsed(TimeUnit.MILLISECONDS) > recheckIntervalMillis) {
+                watch.reset().start();
+                return true;
+            }
+
+            if (log.isDebugEnabled()) {
+                log.debug("Ignoring index [{}] which is not working correctly 
{}", path, getStats());
+            }
+            return false;
+        }
+
+        public String getStats() {
+            return String.format("since %s ,%d indexing cycles, accessed %d 
times",
+                    created, getCycleCount(), accessCount);
+        }
+
+        public int getFailedAccessCount() {
+            return failedAccessCount;
+        }
+
+        public int getAccessCount() {
+            return accessCount;
+        }
+
+        public String getException() {
+            return exception;
+        }
+
+        public long getCreatedTime() {
+            return createdTime;
+        }
+
+        public boolean isPersistedIndex() {
+            return persistedIndex;
+        }
+
+        private int getCycleCount() {
+            return indexerCycleCount - lastIndexerCycleCount;
+        }
+
+        public void failedAccess(Throwable e) {
+            failedAccessCount++;
+            exception = Throwables.getStackTraceAsString(e);
+        }
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/BadIndexTracker.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java?rev=1828972&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
 Thu Apr 12 11:46:17 2018
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.search;
+
+
+public interface FulltextIndexConstants {
+
+    enum IndexingMode {
+        SYNC,
+        NRT,
+        ASYNC;
+
+        public String asyncValueName(){
+            return name().toLowerCase();
+        }
+
+        public static IndexingMode from(String indexingMode){
+            return valueOf(indexingMode.toUpperCase());
+        }
+    }
+
+    String INDEX_DATA_CHILD_NAME = ":data";
+
+    /**
+     * include only certain property types in the index
+     */
+    String INCLUDE_PROPERTY_TYPES = "includePropertyTypes";
+
+    /**
+     * exclude certain properties by name
+     */
+    String EXCLUDE_PROPERTY_NAMES = "excludePropertyNames";
+
+    String PERSISTENCE_NAME = "persistence";
+
+    String PERSISTENCE_OAK = "repository";
+
+    String PERSISTENCE_FILE = "file";
+
+    String PERSISTENCE_PATH = "path";
+
+    /**
+     * Experimental flag to control storage behavior: 'null' or 'true' means 
the content is stored
+     */
+    String EXPERIMENTAL_STORAGE = "oak.experimental.storage";
+
+    /**
+     * Determines if full text indexing is enabled for this index definition.
+     * Default is true
+     */
+    String FULL_TEXT_ENABLED = "fulltextEnabled";
+
+    /**
+     * Only include properties with name in this set. If this property is 
defined
+     * then {@code excludePropertyNames} would be ignored
+     */
+    String INCLUDE_PROPERTY_NAMES = "includePropertyNames";
+
+    /**
+     * Type of the property being indexed defined as part of property 
definition
+     * under the given index definition. Refer to {@link 
javax.jcr.PropertyType}
+     * contants for the possible values
+     */
+    String PROP_TYPE = "type";
+
+    /**
+     * Defines properties which would be used for ordering. If range queries 
are to
+     * be performed with same property then it must be part of include list 
also
+     */
+    String ORDERED_PROP_NAMES = "orderedProps";
+
+    /**
+     * Size in bytes used for splitting the index files when storing them in 
NodeStore
+     */
+    String BLOB_SIZE = "blobSize";
+
+    /**
+     * Native function name associated with this index definition. Any query 
can
+     * use this as the function name to ensure that this index gets used for 
invoking
+     * the index
+     */
+    String FUNC_NAME = "functionName";
+
+    /**
+     * Child node name under which property details are provided
+     */
+    String PROP_NODE = "properties";
+
+    String INDEX_RULES = "indexRules";
+
+    /**
+     * Field boost factor
+     */
+    String FIELD_BOOST = "boost";
+
+    /**
+     * Property name defined explicitly. Mostly used in case of relative 
property names
+     */
+    String PROP_NAME = "name";
+
+    String PROP_IS_REGEX = "isRegexp";
+
+    String PROP_INDEX = "index";
+
+    String PROP_USE_IN_EXCERPT = "useInExcerpt";
+
+    String EXCERPT_NODE_FIELD_NAME = ".";
+
+    String PROP_NODE_SCOPE_INDEX = "nodeScopeIndex";
+
+    String PROP_PROPERTY_INDEX = "propertyIndex";
+
+    String PROP_ANALYZED = "analyzed";
+
+    String RULE_INHERITED = "inherited";
+
+    String PROP_ORDERED = "ordered";
+
+    String PROP_SCORER_PROVIDER = "scorerProviderName";
+
+    String PROP_WEIGHT = "weight";
+
+    /**
+     * Boolean property in property definition to mark sync properties
+     */
+    String PROP_SYNC = "sync";
+
+    /**
+     * Boolean property in property definition to mark unique properties
+     */
+    String PROP_UNIQUE = "unique";
+
+
+    String EVALUATE_PATH_RESTRICTION = "evaluatePathRestrictions";
+
+    /**
+     * Experimental config to restrict which property type gets indexed at
+     * property definition level. Mostly index rule level 
#INCLUDE_PROPERTY_TYPES
+     * should be sufficient
+     */
+    String PROP_INCLUDED_TYPE = "oak.experimental.includePropertyTypes";
+
+    /**
+     * Regex to allow inclusion of all immediate properties of the node
+     */
+    String REGEX_ALL_PROPS = "^[^\\/]*$";
+
+    /**
+     * Node name storing the aggregate rules
+     */
+    String AGGREGATES = "aggregates";
+
+    String AGG_PRIMARY_TYPE = "primaryType";
+
+    /**
+     * Name of property which stores the aggregate include pattern like 
<code>jcr:content/metadata</code>
+     */
+    String AGG_PATH = "path";
+
+    /**
+     * Limit for maximum number of reaggregates allowed. For example if there 
is an aggregate of nt:folder
+     * and it also includes nt:folder then aggregation would traverse down 
untill this limit is hit
+     */
+    String AGG_RECURSIVE_LIMIT = "reaggregateLimit";
+
+    /**
+     * Boolean property indicating that separate fulltext field should be 
created for
+     * node represented by this pattern
+     */
+    String AGG_RELATIVE_NODE = "relativeNode";
+
+    String COST_PER_ENTRY = "costPerEntry";
+
+    String COST_PER_EXECUTION = "costPerExecution";
+
+
+    /**
+     * Config node which include Tika related configuration
+     */
+    String TIKA = "tika";
+
+    /**
+     * nt:file node under 'tika' node which refers to the config xml file
+     */
+    String TIKA_CONFIG = "config.xml";
+
+    String TIKA_MAX_EXTRACT_LENGTH = "maxExtractLength";
+
+    /**
+     *  Config node under tika which defines mime type mappings
+     */
+    String TIKA_MIME_TYPES = "mimeTypes";
+
+    /**
+     * Property name within the mime type structure which defines a mime type 
mapping
+     */
+    String TIKA_MAPPED_TYPE = "mappedType";
+
+    /**
+     * The maximum number of terms that will be indexed for a single field in a
+     * document.  This limits the amount of memory required for indexing, so 
that
+     * collections with very large files will not crash the indexing process by
+     * running out of memory.
+     * <p>
+     * Note that this effectively truncates large documents, excluding from the
+     * index terms that occur further in the document.  If you know your source
+     * documents are large, be sure to set this value high enough to 
accommodate
+     * the expected size.  If you set it to Integer.MAX_VALUE, then the only 
limit
+     * is your memory, but you should anticipate an OutOfMemoryError.
+     * <p>
+     * By default, no more than 10,000 terms will be indexed for a field.
+     */
+    String MAX_FIELD_LENGTH = "maxFieldLength";
+
+    /**
+     * whether use this property values for suggestions
+     */
+    String PROP_USE_IN_SUGGEST = "useInSuggest";
+
+    /**
+     * subnode holding configuration for suggestions
+     */
+    String SUGGESTION_CONFIG = "suggestion";
+
+    /**
+     * update frequency of the suggester in minutes
+     */
+    String SUGGEST_UPDATE_FREQUENCY_MINUTES = "suggestUpdateFrequencyMinutes";
+
+    /**
+     * whether use this property values for spellchecking
+     */
+    String PROP_USE_IN_SPELLCHECK = "useInSpellcheck";
+
+    /**
+     * Property definition config indicating that null check support should be
+     * enabled for this property
+     */
+    String PROP_NULL_CHECK_ENABLED = "nullCheckEnabled";
+
+    /**
+     * Property definition config indicating that this property would be used 
with
+     * 'IS NOT NULL' constraint
+     */
+    String PROP_NOT_NULL_CHECK_ENABLED = "notNullCheckEnabled";
+
+    /**
+     * IndexRule level config to indicate that Node name should also be index
+     * to support fn:name() queries
+     */
+    String INDEX_NODE_NAME = "indexNodeName";
+
+    /**
+     * Property definition name to indicate indexing node name
+     */
+    String PROPDEF_PROP_NODE_NAME = ":nodeName";
+
+
+    /**
+     * Optional subnode holding configuration for facets.
+     */
+    String FACETS = "facets";
+
+    /**
+     * Optional property to set the suggest field to be analyzed and therefore 
allow more fine
+     * grained and flexible suggestions.
+     */
+    String SUGGEST_ANALYZED = "suggestAnalyzed";
+
+    /**
+     * Integer property indicating that LuceneIndex should be
+     * used in compat mode to specific version
+     */
+    String COMPAT_MODE = "compatVersion";
+
+    /**
+     * Optional (index definition) property indicating max number of facets 
that will be retrieved
+     * in query
+     * Default is {@link IndexDefinition#DEFAULT_FACET_COUNT}
+     */
+    String PROP_FACETS_TOP_CHILDREN = "topChildren";
+
+    /**
+     * Optional (property definition) property indicating whether facets 
should be created
+     * for this property
+     */
+    String PROP_FACETS = "facets";
+
+    /**
+     * Boolean property indicate that property should not be included in 
aggregation
+     */
+    String PROP_EXCLUDE_FROM_AGGREGATE = "excludeFromAggregation";
+
+    /**
+     * String property: the function to index, for function-based index
+     */
+    String PROP_FUNCTION = "function";
+
+    /**
+     * Boolean property which signal LuceneIndexEditor to refresh the stored 
index definition
+     */
+    String PROP_REFRESH_DEFN = "refresh";
+
+    /**
+     * Boolean property to indicate that nodes nodetype matching indexRule name
+     * should be indexed
+     */
+    String PROP_INDEX_NODE_TYPE = "nodeTypeIndex";
+}

Propchange: 
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
------------------------------------------------------------------------------
    svn:eol-style = native


Reply via email to