Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java?rev=1841926&r1=1841925&r2=1841926&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java Tue Sep 25 12:24:15 2018 @@ -35,28 +35,31 @@ import com.google.common.collect.Sets; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.PropertyValue; import org.apache.jackrabbit.oak.api.Result.SizePrecision; -import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule; +import org.apache.jackrabbit.oak.plugins.index.Cursors; +import org.apache.jackrabbit.oak.plugins.index.Cursors.PathCursor; import org.apache.jackrabbit.oak.plugins.index.lucene.util.MoreLikeThisHelper; import org.apache.jackrabbit.oak.plugins.index.lucene.util.PathStoredFieldVisitor; import org.apache.jackrabbit.oak.plugins.index.lucene.util.SpellcheckHelper; import org.apache.jackrabbit.oak.plugins.index.lucene.util.SuggestHelper; +import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.IndexingRule; +import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.SizeEstimator; import org.apache.jackrabbit.oak.plugins.memory.PropertyValues; -import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextAnd; -import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextContains; -import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextExpression; -import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextOr; -import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextTerm; -import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextVisitor; import org.apache.jackrabbit.oak.spi.query.Cursor; -import org.apache.jackrabbit.oak.plugins.index.Cursors; -import org.apache.jackrabbit.oak.plugins.index.Cursors.PathCursor; import org.apache.jackrabbit.oak.spi.query.Filter; import org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction; import org.apache.jackrabbit.oak.spi.query.IndexRow; import org.apache.jackrabbit.oak.spi.query.QueryConstants; -import org.apache.jackrabbit.oak.spi.query.QueryIndex; import org.apache.jackrabbit.oak.spi.query.QueryIndex.AdvanceFulltextQueryIndex; import org.apache.jackrabbit.oak.spi.query.QueryLimits; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextAnd; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextContains; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextExpression; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextOr; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextTerm; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextVisitor; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; @@ -69,7 +72,6 @@ import org.apache.lucene.index.MultiFiel import org.apache.lucene.index.Term; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; -import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; @@ -99,15 +101,23 @@ import static com.google.common.base.Pre import static org.apache.jackrabbit.JcrConstants.JCR_MIXINTYPES; import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE; import static org.apache.jackrabbit.oak.api.Type.STRING; -import static org.apache.jackrabbit.oak.commons.PathUtils.*; +import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot; +import static org.apache.jackrabbit.oak.commons.PathUtils.getAncestorPath; +import static org.apache.jackrabbit.oak.commons.PathUtils.getDepth; +import static org.apache.jackrabbit.oak.commons.PathUtils.getName; +import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newFulltextTerm; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newPathTerm; -import static org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper.skipTokenization; +import static org.apache.jackrabbit.oak.plugins.index.search.util.IndexHelper.skipTokenization; import static org.apache.jackrabbit.oak.spi.query.QueryConstants.JCR_PATH; -import static org.apache.lucene.search.BooleanClause.Occur.*; +import static org.apache.lucene.search.BooleanClause.Occur.MUST; +import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT; +import static org.apache.lucene.search.BooleanClause.Occur.SHOULD; /** + * Used to query old (compatVersion 1) Lucene indexes. + * * Provides a QueryIndex that does lookups against a Lucene-based index * * <p> @@ -129,6 +139,8 @@ import static org.apache.lucene.search.B * </ul> * <p> * <pre>{@code + * <pre>{@code + * { * { * NodeBuilder index = root.child("oak:index"); * index.child("lucene") @@ -195,7 +207,7 @@ public class LuceneIndex implements Adva return Collections.emptyList(); } - String indexPath = new LuceneIndexLookup(rootState).getOldFullTextIndexPath(filter, tracker); + String indexPath = LuceneIndexLookupUtil.getOldFullTextIndexPath(rootState, filter, tracker); if (indexPath == null) { // unusable index return Collections.emptyList(); } @@ -206,7 +218,7 @@ public class LuceneIndex implements Adva // "contains(a/x, 'hello') and contains(b/x, 'world')" return Collections.emptyList(); } - IndexNode node = tracker.acquireIndexNode(indexPath); + LuceneIndexNode node = tracker.acquireIndexNode(indexPath); try{ if (node != null){ IndexDefinition defn = node.getDefinition(); @@ -241,7 +253,7 @@ public class LuceneIndex implements Adva @Override public String getPlanDescription(IndexPlan plan, NodeState root) { Filter filter = plan.getFilter(); - IndexNode index = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH)); + LuceneIndexNode index = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH)); checkState(index != null, "The Lucene index is not available"); try { FullTextExpression ft = filter.getFullTextConstraint(); @@ -346,7 +358,7 @@ public class LuceneIndex implements Adva ScoreDoc lastDocToRecord = null; - IndexNode indexNode = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH)); + LuceneIndexNode indexNode = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH)); checkState(indexNode != null); try { IndexSearcher searcher = indexNode.getSearcher(); @@ -480,7 +492,7 @@ public class LuceneIndex implements Adva SizeEstimator sizeEstimator = new SizeEstimator() { @Override public long getSize() { - IndexNode indexNode = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH)); + LuceneIndexNode indexNode = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH)); checkState(indexNode != null); try { IndexSearcher searcher = indexNode.getSearcher(); @@ -602,8 +614,8 @@ public class LuceneIndex implements Adva * @param indexDefinition nodestate that contains the index definition * @return the Lucene query */ - private static LuceneRequestFacade getLuceneRequest(Filter filter, IndexReader reader, - boolean nonFullTextConstraints, IndexDefinition indexDefinition) { + private static LuceneRequestFacade getLuceneRequest(Filter filter, IndexReader reader, boolean nonFullTextConstraints, + LuceneIndexDefinition indexDefinition) { List<Query> qs = new ArrayList<Query>(); Analyzer analyzer = indexDefinition.getAnalyzer(); FullTextExpression ft = filter.getFullTextConstraint(); @@ -656,7 +668,7 @@ public class LuceneIndex implements Adva } private static void addNonFullTextConstraints(List<Query> qs, - Filter filter, IndexReader reader, Analyzer analyzer, IndexDefinition indexDefinition) { + Filter filter, IndexReader reader, Analyzer analyzer, IndexDefinition indexDefinition) { if (!filter.matchesAllTypes()) { addNodeTypeConstraints(qs, filter); } @@ -807,7 +819,7 @@ public class LuceneIndex implements Adva } private static boolean isExcludedProperty(PropertyRestriction pr, - IndexingRule rule) { + IndexingRule rule) { String name = pr.propertyName; if (name.contains("/")) { // lucene cannot handle child-level property restrictions @@ -904,7 +916,7 @@ public class LuceneIndex implements Adva if (x instanceof BooleanQuery) { BooleanQuery bq = (BooleanQuery) x; if ((bq.getClauses().length == 1) && - (bq.getClauses()[0].getOccur() == BooleanClause.Occur.MUST_NOT)) { + (bq.getClauses()[0].getOccur() == Occur.MUST_NOT)) { hasMustNot = true; q.add(bq.getClauses()[0]); }
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1841926&r1=1841925&r2=1841926&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java Tue Sep 25 12:24:15 2018 @@ -16,30 +16,18 @@ */ package org.apache.jackrabbit.oak.plugins.index.lucene; +import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.util.AbstractAnalysisFactory; import org.apache.lucene.util.Version; +/** + * Constants used internally in Lucene indexes. + */ public interface LuceneIndexConstants { - enum IndexingMode { - SYNC, - NRT, - ASYNC; - - public String asyncValueName(){ - return name().toLowerCase(); - } - - public static IndexingMode from(String indexingMode){ - return valueOf(indexingMode.toUpperCase()); - } - } - String TYPE_LUCENE = "lucene"; - String INDEX_DATA_CHILD_NAME = ":data"; - String SUGGEST_DATA_CHILD_NAME = ":suggest-data"; String TRASH_CHILD_NAME = ":trash"; @@ -49,66 +37,6 @@ public interface LuceneIndexConstants { Analyzer ANALYZER = new OakAnalyzer(VERSION); /** - * include only certain property types in the index - */ - String INCLUDE_PROPERTY_TYPES = "includePropertyTypes"; - - /** - * exclude certain properties by name - */ - String EXCLUDE_PROPERTY_NAMES = "excludePropertyNames"; - - String PERSISTENCE_NAME = "persistence"; - - String PERSISTENCE_OAK = "repository"; - - String PERSISTENCE_FILE = "file"; - - String PERSISTENCE_PATH = "path"; - - /** - * Experimental flag to control storage behavior: 'null' or 'true' means the content is stored - */ - String EXPERIMENTAL_STORAGE = "oak.experimental.storage"; - - /** - * Determines if full text indexing is enabled for this index definition. - * Default is true - */ - String FULL_TEXT_ENABLED = "fulltextEnabled"; - - /** - * Only include properties with name in this set. If this property is defined - * then {@code excludePropertyNames} would be ignored - */ - String INCLUDE_PROPERTY_NAMES = "includePropertyNames"; - - /** - * Type of the property being indexed defined as part of property definition - * under the given index definition. Refer to {@link javax.jcr.PropertyType} - * contants for the possible values - */ - String PROP_TYPE = "type"; - - /** - * Defines properties which would be used for ordering. If range queries are to - * be performed with same property then it must be part of include list also - */ - String ORDERED_PROP_NAMES = "orderedProps"; - - /** - * Size in bytes used for splitting the index files when storing them in NodeStore - */ - String BLOB_SIZE = "blobSize"; - - /** - * Native function name associated with this index definition. Any query can - * use this as the function name to ensure that this index gets used for invoking - * the index - */ - String FUNC_NAME = "functionName"; - - /** * Name of the codec to be used for indexing */ String CODEC_NAME = "codec"; @@ -119,108 +47,11 @@ public interface LuceneIndexConstants { String MERGE_POLICY_NAME = "mergePolicy"; /** - * Child node name under which property details are provided - */ - String PROP_NODE = "properties"; - - String INDEX_RULES = "indexRules"; - - /** - * Field boost factor - */ - String FIELD_BOOST = "boost"; - - /** - * Property name defined explicitly. Mostly used in case of relative property names - */ - String PROP_NAME = "name"; - - String PROP_IS_REGEX = "isRegexp"; - - String PROP_INDEX = "index"; - - String PROP_USE_IN_EXCERPT = "useInExcerpt"; - - String EXCERPT_NODE_FIELD_NAME = "."; - - String PROP_NODE_SCOPE_INDEX = "nodeScopeIndex"; - - String PROP_PROPERTY_INDEX = "propertyIndex"; - - String PROP_ANALYZED = "analyzed"; - - String RULE_INHERITED = "inherited"; - - String PROP_ORDERED = "ordered"; - - String PROP_SCORER_PROVIDER = "scorerProviderName"; - - String PROP_WEIGHT = "weight"; - - /** - * Boolean property in property definition to mark sync properties - */ - String PROP_SYNC = "sync"; - - /** - * Boolean property in property definition to mark unique properties - */ - String PROP_UNIQUE = "unique"; - - /** - * Integer property indicating that LuceneIndex should be - * used in compat mode to specific version - */ - String COMPAT_MODE = "compatVersion"; - - /** * Boolean property to indicate that LuceneIndex is being used in testMode * and it should participate in every test */ String TEST_MODE = "testMode"; - String EVALUATE_PATH_RESTRICTION = "evaluatePathRestrictions"; - - /** - * Experimental config to restrict which property type gets indexed at - * property definition level. Mostly index rule level #INCLUDE_PROPERTY_TYPES - * should be sufficient - */ - String PROP_INCLUDED_TYPE = "oak.experimental.includePropertyTypes"; - - /** - * Regex to allow inclusion of all immediate properties of the node - */ - String REGEX_ALL_PROPS = "^[^\\/]*$"; - - /** - * Node name storing the aggregate rules - */ - String AGGREGATES = "aggregates"; - - String AGG_PRIMARY_TYPE = "primaryType"; - - /** - * Name of property which stores the aggregate include pattern like <code>jcr:content/metadata</code> - */ - String AGG_PATH = "path"; - - /** - * Limit for maximum number of reaggregates allowed. For example if there is an aggregate of nt:folder - * and it also includes nt:folder then aggregation would traverse down untill this limit is hit - */ - String AGG_RECURSIVE_LIMIT = "reaggregateLimit"; - - /** - * Boolean property indicating that separate fulltext field should be created for - * node represented by this pattern - */ - String AGG_RELATIVE_NODE = "relativeNode"; - - String COST_PER_ENTRY = "costPerEntry"; - - String COST_PER_EXECUTION = "costPerExecution"; - /** * Boolean property indicating if in-built analyzer should preserve original term * (i.e. use @@ -351,54 +182,4 @@ public interface LuceneIndexConstants { @Deprecated String INDEX_PATH = "indexPath"; - /** - * Optional subnode holding configuration for facets. - */ - String FACETS = "facets"; - - /** - * Optional property to set the suggest field to be analyzed and therefore allow more fine - * grained and flexible suggestions. - */ - String SUGGEST_ANALYZED = "suggestAnalyzed"; - - /** - * Optional (index definition) property indicating whether facets should be ACL checked. - * Default is true - */ - String PROP_SECURE_FACETS = "secure"; - - /** - * Optional (index definition) property indicating max number of facets that will be retrieved - * in query - * Default is {@link IndexDefinition#DEFAULT_FACET_COUNT} - */ - String PROP_FACETS_TOP_CHILDREN = "topChildren"; - - /** - * Optional (property definition) property indicating whether facets should be created - * for this property - */ - String PROP_FACETS = "facets"; - - /** - * Boolean property indicate that property should not be included in aggregation - */ - String PROP_EXCLUDE_FROM_AGGREGATE = "excludeFromAggregation"; - - /** - * String property: the function to index, for function-based index - */ - String PROP_FUNCTION = "function"; - - /** - * Boolean property which signal LuceneIndexEditor to refresh the stored index definition - */ - String PROP_REFRESH_DEFN = "refresh"; - - /** - * Boolean property to indicate that nodes nodetype matching indexRule name - * should be indexed - */ - String PROP_INDEX_NODE_TYPE = "nodeTypeIndex"; } Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexDefinition.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexDefinition.java?rev=1841926&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexDefinition.java (added) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexDefinition.java Tue Sep 25 12:24:15 2018 @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.util.Collections; +import java.util.Map; + +import com.google.common.collect.ImmutableMap; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.TokenizerChain; +import org.apache.jackrabbit.oak.plugins.index.lucene.writer.CommitMitigatingTieredMergePolicy; +import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.IndexFormatVersion; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer; +import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; +import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.index.LogByteSizeMergePolicy; +import org.apache.lucene.index.LogDocMergePolicy; +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.TieredMergePolicy; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.google.common.collect.Maps.newHashMap; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ANL_DEFAULT; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INDEX_ORIGINAL_TERM; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION; +import static org.apache.jackrabbit.oak.plugins.index.search.util.ConfigUtil.getOptionalValue; + +public class LuceneIndexDefinition extends IndexDefinition { + private static final Logger log = LoggerFactory.getLogger(IndexDefinition.class); + + private final boolean saveDirListing; + + private final Map<String, Analyzer> analyzers; + private final Analyzer analyzer; + + private final Codec codec; + + private final int maxFieldLength; + + public LuceneIndexDefinition(NodeState root, NodeState defn, String indexPath) { + this(root, getIndexDefinitionState(defn), determineIndexFormatVersion(defn), determineUniqueId(defn), indexPath); + } + + LuceneIndexDefinition(NodeState root, NodeState defn, IndexFormatVersion version, String uid, String indexPath) { + super(root, defn, version, uid, indexPath); + + this.saveDirListing = getOptionalValue(defn, LuceneIndexConstants.SAVE_DIR_LISTING, true); + this.maxFieldLength = getOptionalValue(defn, LuceneIndexConstants.MAX_FIELD_LENGTH, DEFAULT_MAX_FIELD_LENGTH); + this.analyzers = collectAnalyzers(defn); + this.analyzer = createAnalyzer(); + this.codec = createCodec(); + } + + public static Builder newBuilder(NodeState root, NodeState defn, String indexPath){ + return (Builder)new Builder() + .root(root) + .defn(defn) + .indexPath(indexPath); + } + + public static class Builder extends IndexDefinition.Builder { + @Override + public LuceneIndexDefinition build() { + return (LuceneIndexDefinition)super.build(); + } + + @Override + public LuceneIndexDefinition.Builder reindex() { + super.reindex(); + return this; + } + + @Override + protected IndexDefinition createInstance(NodeState indexDefnStateToUse) { + return new LuceneIndexDefinition(root, indexDefnStateToUse, version, uid, indexPath); + } + } + + @Override + protected String getDefaultFunctionName() { + return "lucene";//TODO should this be LuceneIndexConstants.TYPE_LUCENE? + } + + @Override + protected double getDefaultCostPerEntry(IndexFormatVersion version) { + //For older format cost per entry would be higher as it does a runtime + //aggregation + return version == IndexFormatVersion.V1 ? 1.5 : 1.0; + } + + public boolean saveDirListing() { + return saveDirListing; + } + + @Nullable + public Codec getCodec() { + return codec; + } + + @NotNull + public MergePolicy getMergePolicy() { + // MP is not cached to avoid complaining about multiple IWs with multiplexing writers + return createMergePolicy(); + } + + public Analyzer getAnalyzer(){ + return analyzer; + } + + //~---------------------------------------------------< Analyzer > + + private Analyzer createAnalyzer() { + Analyzer result; + Analyzer defaultAnalyzer = LuceneIndexConstants.ANALYZER; + if (analyzers.containsKey(ANL_DEFAULT)){ + defaultAnalyzer = analyzers.get(ANL_DEFAULT); + } + if (!evaluatePathRestrictions()){ + result = defaultAnalyzer; + } else { + Map<String, Analyzer> analyzerMap = ImmutableMap.<String, Analyzer>builder() + .put(FieldNames.ANCESTORS, + new TokenizerChain(new PathHierarchyTokenizerFactory(Collections.emptyMap()))) + .build(); + result = new PerFieldAnalyzerWrapper(defaultAnalyzer, analyzerMap); + } + + //In case of negative value no limits would be applied + if (maxFieldLength < 0){ + return result; + } + return new LimitTokenCountAnalyzer(result, maxFieldLength); + } + + private static Map<String, Analyzer> collectAnalyzers(NodeState defn) { + Map<String, Analyzer> analyzerMap = newHashMap(); + NodeStateAnalyzerFactory factory = new NodeStateAnalyzerFactory(VERSION); + NodeState analyzersTree = defn.getChildNode(LuceneIndexConstants.ANALYZERS); + for (ChildNodeEntry cne : analyzersTree.getChildNodeEntries()) { + Analyzer a = factory.createInstance(cne.getNodeState()); + analyzerMap.put(cne.getName(), a); + } + + if (getOptionalValue(analyzersTree, INDEX_ORIGINAL_TERM, false) && !analyzerMap.containsKey(ANL_DEFAULT)) { + analyzerMap.put(ANL_DEFAULT, new OakAnalyzer(VERSION, true)); + } + + return ImmutableMap.copyOf(analyzerMap); + } + + + //~---------------------------------------------< utility > + + private Codec createCodec() { + String codecName = getOptionalValue(definition, LuceneIndexConstants.CODEC_NAME, null); + Codec codec = null; + if (codecName != null) { + // prevent LUCENE-6482 + // (also done in LuceneIndexProviderService, just to be save) + OakCodec ensureLucene46CodecLoaded = new OakCodec(); + // to ensure the JVM doesn't optimize away object creation + // (probably not really needed; just to be save) + log.debug("Lucene46Codec is loaded: {}", ensureLucene46CodecLoaded); + codec = Codec.forName(codecName); + log.debug("Codec is loaded: {}", codecName); + } else if (fullTextEnabled) { + codec = new OakCodec(); + } + return codec; + } + + private MergePolicy createMergePolicy() { + String mmp = System.getProperty("oak.lucene.cmmp"); + if (mmp != null) { + return new CommitMitigatingTieredMergePolicy(); + } + + String mergePolicyName = getOptionalValue(definition, LuceneIndexConstants.MERGE_POLICY_NAME, null); + MergePolicy mergePolicy = null; + if (mergePolicyName != null) { + if (mergePolicyName.equalsIgnoreCase("no")) { + mergePolicy = NoMergePolicy.COMPOUND_FILES; + } else if (mergePolicyName.equalsIgnoreCase("mitigated")) { + mergePolicy = new CommitMitigatingTieredMergePolicy(); + } else if (mergePolicyName.equalsIgnoreCase("tiered") || mergePolicyName.equalsIgnoreCase("default")) { + mergePolicy = new TieredMergePolicy(); + } else if (mergePolicyName.equalsIgnoreCase("logbyte")) { + mergePolicy = new LogByteSizeMergePolicy(); + } else if (mergePolicyName.equalsIgnoreCase("logdoc")) { + mergePolicy = new LogDocMergePolicy(); + } + } + if (mergePolicy == null) { + mergePolicy = new TieredMergePolicy(); + } + return mergePolicy; + } +} Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexDefinition.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java?rev=1841926&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (added) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java Tue Sep 25 12:24:15 2018 @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.IndexEditor; +import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter; +import org.apache.jackrabbit.oak.plugins.index.search.Aggregate; +import org.apache.jackrabbit.oak.plugins.index.search.Aggregate.Matcher; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.PropertyUpdateCallback; +import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; +import org.apache.jackrabbit.oak.spi.commit.Editor; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.lucene.document.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.jackrabbit.oak.commons.PathUtils.concat; + +/** + * {@link IndexEditor} implementation that is responsible for keeping the + * {@link LuceneIndex} up to date + * + * @see LuceneIndex + */ +public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { + + private static final Logger log = + LoggerFactory.getLogger(LuceneIndexEditor.class); + + private final LuceneIndexEditorContext context; + + /** Name of this node, or {@code null} for the root node. */ + private final String name; + + /** Parent editor or {@code null} if this is the root editor. */ + private final LuceneIndexEditor parent; + + /** Path of this editor, built lazily in {@link #getPath()}. */ + private String path; + + private boolean propertiesChanged = false; + + private List<PropertyState> propertiesModified = Lists.newArrayList(); + + /** + * Flag indicating if the current tree being traversed has a deleted parent. + */ + private final boolean isDeleted; + + private IndexDefinition.IndexingRule indexingRule; + + private List<Matcher> currentMatchers = Collections.emptyList(); + + private final MatcherState matcherState; + + private final PathFilter.Result pathFilterResult; + + LuceneIndexEditor(LuceneIndexEditorContext context) throws CommitFailedException { + this.parent = null; + this.name = null; + this.path = "/"; + this.context = context; + this.isDeleted = false; + this.matcherState = MatcherState.NONE; + this.pathFilterResult = context.getDefinition().getPathFilter().filter(PathUtils.ROOT_PATH); + } + + private LuceneIndexEditor(LuceneIndexEditor parent, String name, + MatcherState matcherState, + PathFilter.Result pathFilterResult, + boolean isDeleted) { + this.parent = parent; + this.name = name; + this.path = null; + this.context = parent.context; + this.isDeleted = isDeleted; + this.matcherState = matcherState; + this.pathFilterResult = pathFilterResult; + } + + public String getPath() { + if (path == null) { // => parent != null + path = concat(parent.getPath(), name); + } + return path; + } + + @Override + public void enter(NodeState before, NodeState after) + throws CommitFailedException { + if (EmptyNodeState.MISSING_NODE == before && parent == null){ + context.enableReindexMode(); + } + + //Only check for indexing if the result is include. + //In case like TRAVERSE nothing needs to be indexed for those + //path + if (pathFilterResult == PathFilter.Result.INCLUDE) { + //For traversal in deleted sub tree before state has to be used + NodeState current = after.exists() ? after : before; + indexingRule = getDefinition().getApplicableIndexingRule(current); + + if (indexingRule != null) { + currentMatchers = indexingRule.getAggregate().createMatchers(this); + } + } + } + + @Override + public void leave(NodeState before, NodeState after) + throws CommitFailedException { + if (propertiesChanged || !before.exists()) { + String path = getPath(); + if (addOrUpdate(path, after, before.exists())) { + long indexed = context.incIndexedNodes(); + if (indexed % 1000 == 0) { + log.debug("[{}] => Indexed {} nodes...", getIndexName(), indexed); + } + } + } + + for (Matcher m : matcherState.affectedMatchers){ + m.markRootDirty(); + } + + if (parent == null) { + PropertyUpdateCallback callback = context.getPropertyUpdateCallback(); + if (callback != null) { + callback.done(); + } + + try { + context.closeWriter(); + } catch (IOException e) { + CommitFailedException ce = new CommitFailedException("Lucene", 4, + "Failed to close the Lucene index " + context.getIndexingContext().getIndexPath(), e); + context.getIndexingContext().indexUpdateFailed(ce); + throw ce; + } + if (context.getIndexedNodes() > 0) { + log.debug("[{}] => Indexed {} nodes, done.", getIndexName(), context.getIndexedNodes()); + } + } + } + + @Override + public void propertyAdded(PropertyState after) { + markPropertyChanged(after.getName()); + checkAggregates(after.getName()); + propertyUpdated(null, after); + } + + @Override + public void propertyChanged(PropertyState before, PropertyState after) { + markPropertyChanged(before.getName()); + propertiesModified.add(before); + checkAggregates(before.getName()); + propertyUpdated(before, after); + } + + @Override + public void propertyDeleted(PropertyState before) { + markPropertyChanged(before.getName()); + propertiesModified.add(before); + checkAggregates(before.getName()); + propertyUpdated(before, null); + } + + @Override + public Editor childNodeAdded(String name, NodeState after) { + PathFilter.Result filterResult = getPathFilterResult(name); + if (filterResult != PathFilter.Result.EXCLUDE) { + return new LuceneIndexEditor(this, name, getMatcherState(name, after), filterResult, false); + } + return null; + } + + @Override + public Editor childNodeChanged( + String name, NodeState before, NodeState after) { + PathFilter.Result filterResult = getPathFilterResult(name); + if (filterResult != PathFilter.Result.EXCLUDE) { + return new LuceneIndexEditor(this, name, getMatcherState(name, after), filterResult, false); + } + return null; + } + + @Override + public Editor childNodeDeleted(String name, NodeState before) + throws CommitFailedException { + PathFilter.Result filterResult = getPathFilterResult(name); + if (filterResult == PathFilter.Result.EXCLUDE) { + return null; + } + + if (!isDeleted) { + // tree deletion is handled on the parent node + String path = concat(getPath(), name); + try { + LuceneIndexWriter writer = context.getWriter(); + // Remove all index entries in the removed subtree + writer.deleteDocuments(path); + this.context.indexUpdate(); + } catch (IOException e) { + CommitFailedException ce = new CommitFailedException("Lucene", 5, "Failed to remove the index entries of" + + " the removed subtree " + path + "for index " + context.getIndexingContext().getIndexPath(), e); + context.getIndexingContext().indexUpdateFailed(ce); + throw ce; + } + } + + MatcherState ms = getMatcherState(name, before); + if (!ms.isEmpty()){ + return new LuceneIndexEditor(this, name, ms, filterResult, true); + } + return null; // no need to recurse down the removed subtree + } + + LuceneIndexEditorContext getContext() { + return context; + } + + private boolean addOrUpdate(String path, NodeState state, boolean isUpdate) + throws CommitFailedException { + try { + Document d = makeDocument(path, state, isUpdate); + if (d != null) { + if (log.isTraceEnabled()) { + log.trace("[{}] Indexed document for {} is {}", getIndexName(), path, d); + } + context.indexUpdate(); + context.getWriter().updateDocument(path, d); + return true; + } + } catch (IOException e) { + CommitFailedException ce = new CommitFailedException("Lucene", 3, + "Failed to index the node " + path, e); + context.getIndexingContext().indexUpdateFailed(ce); + throw ce; + } catch (IllegalArgumentException ie) { + log.warn("Failed to index the node [{}]", path, ie); + } + return false; + } + + private Document makeDocument(String path, NodeState state, boolean isUpdate) throws IOException { + if (!isIndexable()) { + return null; + } + return context.newDocumentMaker(indexingRule, path).makeDocument(state, isUpdate, propertiesModified); + } + + + //~-------------------------------------------------------< Aggregate > + + @Override + public void markDirty() { + propertiesChanged = true; + } + + private MatcherState getMatcherState(String name, NodeState after) { + List<Matcher> matched = Lists.newArrayList(); + List<Matcher> inherited = Lists.newArrayList(); + for (Matcher m : Iterables.concat(matcherState.inherited, currentMatchers)) { + Matcher result = m.match(name, after); + if (result.getStatus() == Matcher.Status.MATCH_FOUND){ + matched.add(result); + } + + if (result.getStatus() != Matcher.Status.FAIL){ + inherited.addAll(result.nextSet()); + } + } + + if (!matched.isEmpty() || !inherited.isEmpty()) { + return new MatcherState(matched, inherited); + } + return MatcherState.NONE; + } + + + /** + * Determines which all matchers are affected by this property change + * + * @param name modified property name + */ + private void checkAggregates(String name) { + for (Matcher m : matcherState.matched) { + if (!matcherState.affectedMatchers.contains(m) + && m.aggregatesProperty(name)) { + matcherState.affectedMatchers.add(m); + } + } + } + + private static class MatcherState { + final static MatcherState NONE = new MatcherState(Collections.<Matcher>emptyList(), + Collections.<Matcher>emptyList()); + + final List<Matcher> matched; + final List<Matcher> inherited; + final Set<Matcher> affectedMatchers; + + public MatcherState(List<Matcher> matched, + List<Matcher> inherited){ + this.matched = matched; + this.inherited = inherited; + + //Affected matches would only be used when there are + //some matched matchers + if (matched.isEmpty()){ + affectedMatchers = Collections.emptySet(); + } else { + affectedMatchers = Sets.newIdentityHashSet(); + } + } + + public boolean isEmpty() { + return matched.isEmpty() && inherited.isEmpty(); + } + } + + private void markPropertyChanged(String name) { + if (isIndexable() + && !propertiesChanged + && indexingRule.isIndexed(name)) { + propertiesChanged = true; + } + } + + private void propertyUpdated(PropertyState before, PropertyState after) { + PropertyUpdateCallback callback = context.getPropertyUpdateCallback(); + + //Avoid further work if no callback is present + if (callback == null) { + return; + } + + String propertyName = before != null ? before.getName() : after.getName(); + + if (isIndexable()) { + PropertyDefinition pd = indexingRule.getConfig(propertyName); + if (pd != null) { + callback.propertyUpdated(getPath(), propertyName, pd, before, after); + } + } + + for (Matcher m : matcherState.matched) { + if (m.aggregatesProperty(propertyName)) { + Aggregate.Include i = m.getCurrentInclude(); + if (i instanceof Aggregate.PropertyInclude) { + PropertyDefinition pd = ((Aggregate.PropertyInclude) i).getPropertyDefinition(); + String propertyRelativePath = PathUtils.concat(m.getMatchedPath(), propertyName); + + callback.propertyUpdated(m.getRootPath(), propertyRelativePath, pd, before, after); + } + } + } + } + + private IndexDefinition getDefinition() { + return context.getDefinition(); + } + + private boolean isIndexable(){ + return indexingRule != null; + } + + private PathFilter.Result getPathFilterResult(String childNodeName) { + return context.getDefinition().getPathFilter().filter(concat(getPath(), childNodeName)); + } + + private String getIndexName() { + return context.getDefinition().getIndexName(); + } +} Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java?rev=1841926&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java (added) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java Tue Sep 25 12:24:15 2018 @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.io.IOException; + +import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.IndexingContext; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.FacetHelper; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.FacetsConfigProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter; +import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexEditorContext; +import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriterFactory; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.stats.Clock; +import org.apache.lucene.facet.FacetsConfig; +import org.jetbrains.annotations.Nullable; + +public class LuceneIndexEditorContext extends FulltextIndexEditorContext implements FacetsConfigProvider { + private FacetsConfig facetsConfig; + + private final IndexAugmentorFactory augmentorFactory; + + LuceneIndexEditorContext(NodeState root, NodeBuilder definition, + @Nullable IndexDefinition indexDefinition, + IndexUpdateCallback updateCallback, + FulltextIndexWriterFactory indexWriterFactory, + ExtractedTextCache extractedTextCache, + IndexAugmentorFactory augmentorFactory, + IndexingContext indexingContext, boolean asyncIndexing) { + super(root, definition, indexDefinition, updateCallback, indexWriterFactory, extractedTextCache, + indexingContext, asyncIndexing); + this.augmentorFactory = augmentorFactory; + } + + @Override + public IndexDefinition.Builder newDefinitionBuilder() { + return new LuceneIndexDefinition.Builder(); + } + + @Override + public LuceneDocumentMaker newDocumentMaker(IndexDefinition.IndexingRule rule, String path){ + //Faceting is only enabled for async mode + FacetsConfigProvider facetsConfigProvider = isAsyncIndexing() ? this : null; + return new LuceneDocumentMaker(getTextExtractor(), facetsConfigProvider, augmentorFactory, + definition, rule, path); + } + + @Override + public LuceneIndexWriter getWriter() throws IOException { + return (LuceneIndexWriter)super.getWriter(); + } + + @Override + public FacetsConfig getFacetsConfig() { + if (facetsConfig == null){ + facetsConfig = FacetHelper.getFacetsConfig(definitionBuilder); + } + return facetsConfig; + } + + /** Only set for testing */ + public static void setClock(Clock c) { + FulltextIndexEditorContext.setClock(c); + } +} \ No newline at end of file Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java?rev=1841926&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java (added) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java Tue Sep 25 12:24:15 2018 @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.plugins.index.ContextAwareCallback; +import org.apache.jackrabbit.oak.plugins.index.IndexEditor; +import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.IndexingContext; +import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollector; +import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory.BlobDeletionCallback; +import org.apache.jackrabbit.oak.plugins.index.lucene.directory.DefaultDirectoryFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.IndexingQueue; +import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexWriterFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LuceneDocumentHolder; +import org.apache.jackrabbit.oak.plugins.index.lucene.property.LuceneIndexPropertyQuery; +import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyQuery; +import org.apache.jackrabbit.oak.plugins.index.lucene.writer.DefaultIndexWriterFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterConfig; +import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriterFactory; +import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; +import org.apache.jackrabbit.oak.spi.commit.CommitContext; +import org.apache.jackrabbit.oak.spi.commit.Editor; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; +import org.apache.jackrabbit.oak.spi.mount.Mounts; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE; + +/** + * Service that provides Lucene based {@link IndexEditor}s + * + * @see LuceneIndexEditor + * @see IndexEditorProvider + * + */ +public class LuceneIndexEditorProvider implements IndexEditorProvider { + private final Logger log = LoggerFactory.getLogger(getClass()); + private final IndexCopier indexCopier; + private final ExtractedTextCache extractedTextCache; + private final IndexAugmentorFactory augmentorFactory; + private final IndexTracker indexTracker; + private final MountInfoProvider mountInfoProvider; + private final ActiveDeletedBlobCollector activeDeletedBlobCollector; + private GarbageCollectableBlobStore blobStore; + private IndexingQueue indexingQueue; + private boolean nrtIndexingEnabled; + private LuceneIndexWriterConfig writerConfig = new LuceneIndexWriterConfig(); + + /** + * Number of indexed Lucene document that can be held in memory + * This ensures that for very large commit memory consumption + * is bounded + */ + private int inMemoryDocsLimit = Integer.getInteger("oak.lucene.inMemoryDocsLimit", 500); + + public LuceneIndexEditorProvider() { + this(null); + } + + public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier) { + //Disable the cache by default in ExtractedTextCache + this(indexCopier, new ExtractedTextCache(0, 0)); + } + + public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier, + ExtractedTextCache extractedTextCache) { + this(indexCopier, extractedTextCache, null, Mounts.defaultMountInfoProvider()); + } + + public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier, + ExtractedTextCache extractedTextCache, + @Nullable IndexAugmentorFactory augmentorFactory, + MountInfoProvider mountInfoProvider) { + this(indexCopier, null, extractedTextCache, augmentorFactory, mountInfoProvider); + } + + public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier, + @Nullable IndexTracker indexTracker, + ExtractedTextCache extractedTextCache, + @Nullable IndexAugmentorFactory augmentorFactory, + MountInfoProvider mountInfoProvider) { + this(indexCopier, indexTracker, extractedTextCache, augmentorFactory, mountInfoProvider, + ActiveDeletedBlobCollectorFactory.NOOP); + } + public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier, + @Nullable IndexTracker indexTracker, + ExtractedTextCache extractedTextCache, + @Nullable IndexAugmentorFactory augmentorFactory, + MountInfoProvider mountInfoProvider, + @NotNull ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollector activeDeletedBlobCollector) { + this.indexCopier = indexCopier; + this.indexTracker = indexTracker; + this.extractedTextCache = extractedTextCache != null ? extractedTextCache : new ExtractedTextCache(0, 0); + this.augmentorFactory = augmentorFactory; + this.mountInfoProvider = checkNotNull(mountInfoProvider); + this.activeDeletedBlobCollector = activeDeletedBlobCollector; + } + + @Override + public Editor getIndexEditor( + @NotNull String type, @NotNull NodeBuilder definition, @NotNull NodeState root, + @NotNull IndexUpdateCallback callback) + throws CommitFailedException { + if (TYPE_LUCENE.equals(type)) { + checkArgument(callback instanceof ContextAwareCallback, "callback instance not of type " + + "ContextAwareCallback [%s]", callback); + IndexingContext indexingContext = ((ContextAwareCallback)callback).getIndexingContext(); + BlobDeletionCallback blobDeletionCallback = activeDeletedBlobCollector.getBlobDeletionCallback(); + indexingContext.registerIndexCommitCallback(blobDeletionCallback); + FulltextIndexWriterFactory writerFactory = null; + LuceneIndexDefinition indexDefinition = null; + boolean asyncIndexing = true; + String indexPath = indexingContext.getIndexPath(); + PropertyIndexUpdateCallback propertyUpdateCallback = null; + + if (nrtIndexingEnabled() && !indexingContext.isAsync() && IndexDefinition.supportsSyncOrNRTIndexing(definition)) { + + //Would not participate in reindexing. Only interested in + //incremental indexing + if (indexingContext.isReindexing()){ + return null; + } + + CommitContext commitContext = getCommitContext(indexingContext); + if (commitContext == null){ + //Logically there should not be any commit without commit context. But + //some initializer code does the commit with out it. So ignore such calls with + //warning now + //TODO Revisit use of warn level once all such cases are analyzed + log.warn("No CommitContext found for commit", new Exception()); + return null; + } + + //TODO Also check if index has been done once + + + writerFactory = new LocalIndexWriterFactory(getDocumentHolder(commitContext), + indexPath); + + //IndexDefinition from tracker might differ from one passed here for reindexing + //case which should be fine. However reusing existing definition would avoid + //creating definition instance for each commit as this gets executed for each commit + if (indexTracker != null){ + indexDefinition = indexTracker.getIndexDefinition(indexPath); + if (indexDefinition != null && !indexDefinition.hasMatchingNodeTypeReg(root)){ + log.debug("Detected change in NodeType registry for index {}. Would not use " + + "existing index definition", indexDefinition.getIndexPath()); + indexDefinition = null; + } + } + + if (indexDefinition == null) { + indexDefinition = LuceneIndexDefinition.newBuilder(root, definition.getNodeState(), + indexPath).build(); + } + + if (indexDefinition.hasSyncPropertyDefinitions()) { + propertyUpdateCallback = new PropertyIndexUpdateCallback(indexPath, definition, root); + if (indexTracker != null) { + PropertyQuery query = new LuceneIndexPropertyQuery(indexTracker, indexPath); + propertyUpdateCallback.getUniquenessConstraintValidator().setSecondStore(query); + } + } + + //Pass on a read only builder to ensure that nothing gets written + //at all to NodeStore for local indexing. + //TODO [hybrid] This would cause issue with Facets as for faceted fields + //some stuff gets written to NodeBuilder. That logic should be refactored + //to be moved to LuceneIndexWriter + definition = new ReadOnlyBuilder(definition.getNodeState()); + + asyncIndexing = false; + } + + if (writerFactory == null) { + writerFactory = new DefaultIndexWriterFactory(mountInfoProvider, newDirectoryFactory(blobDeletionCallback), writerConfig); + } + + LuceneIndexEditorContext context = new LuceneIndexEditorContext(root, definition, indexDefinition, callback, + writerFactory, extractedTextCache, augmentorFactory, indexingContext, asyncIndexing); + + context.setPropertyUpdateCallback(propertyUpdateCallback); + return new LuceneIndexEditor(context); + } + return null; + } + + IndexCopier getIndexCopier() { + return indexCopier; + } + + IndexingQueue getIndexingQueue() { + return indexingQueue; + } + + public ExtractedTextCache getExtractedTextCache() { + return extractedTextCache; + } + + public void setInMemoryDocsLimit(int inMemoryDocsLimit) { + this.inMemoryDocsLimit = inMemoryDocsLimit; + } + + protected DirectoryFactory newDirectoryFactory(BlobDeletionCallback blobDeletionCallback) { + return new DefaultDirectoryFactory(indexCopier, blobStore, blobDeletionCallback); + } + + private LuceneDocumentHolder getDocumentHolder(CommitContext commitContext){ + LuceneDocumentHolder holder = (LuceneDocumentHolder) commitContext.get(LuceneDocumentHolder.NAME); + if (holder == null) { + holder = new LuceneDocumentHolder(indexingQueue, inMemoryDocsLimit); + commitContext.set(LuceneDocumentHolder.NAME, holder); + } + return holder; + } + + public void setBlobStore(@Nullable GarbageCollectableBlobStore blobStore) { + this.blobStore = blobStore; + } + + public void setIndexingQueue(IndexingQueue indexingQueue) { + this.indexingQueue = indexingQueue; + this.nrtIndexingEnabled = indexingQueue != null; + } + + public void setWriterConfig(LuceneIndexWriterConfig writerConfig) { + this.writerConfig = writerConfig; + } + + GarbageCollectableBlobStore getBlobStore() { + return blobStore; + } + + private boolean nrtIndexingEnabled() { + return nrtIndexingEnabled; + } + + private static CommitContext getCommitContext(IndexingContext indexingContext) { + return (CommitContext) indexingContext.getCommitInfo().getInfo().get(CommitContext.NAME); + } +} Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java?rev=1841926&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java (added) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java Tue Sep 25 12:24:15 2018 @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.io.File; +import java.io.IOException; +import java.util.Set; + +import com.google.common.collect.ImmutableSet; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.json.JsopDiff; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfo; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.IndexInfo; +import org.apache.jackrabbit.oak.plugins.index.IndexInfoProvider; +import org.apache.jackrabbit.oak.plugins.index.IndexUtils; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryUtils; +import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker; +import org.apache.jackrabbit.oak.plugins.index.lucene.directory.OakDirectory; +import org.apache.jackrabbit.oak.plugins.index.lucene.writer.MultiplexersLucene; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.util.NodeStateCloner; +import org.apache.jackrabbit.oak.spi.state.EqualsDiff; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder; +import org.apache.jackrabbit.util.ISO8601; +import org.apache.lucene.store.Directory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE; + +public class LuceneIndexInfoProvider implements IndexInfoProvider { + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final NodeStore nodeStore; + + private final AsyncIndexInfoService asyncInfoService; + + private final File workDir; + + public LuceneIndexInfoProvider(NodeStore nodeStore, AsyncIndexInfoService asyncInfoService, File workDir) { + this.nodeStore = checkNotNull(nodeStore); + this.asyncInfoService = checkNotNull(asyncInfoService); + this.workDir = checkNotNull(workDir); + } + + @Override + public String getType() { + return LuceneIndexConstants.TYPE_LUCENE; + } + + @Override + public IndexInfo getInfo(String indexPath) throws IOException { + NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(), indexPath); + + checkArgument(LuceneIndexConstants.TYPE_LUCENE.equals(idxState.getString(IndexConstants.TYPE_PROPERTY_NAME)), + "Index definition at [%s] is not of type 'lucene'", indexPath); + + LuceneIndexInfo info = new LuceneIndexInfo(indexPath); + computeSize(idxState, info); + computeIndexDefinitionChange(idxState, info); + computeLastUpdatedTime(idxState, info); + computeAsyncIndexInfo(idxState, indexPath, info); + return info; + } + + @Override + public boolean isValid(String indexPath) throws IOException { + IndexConsistencyChecker checker = new IndexConsistencyChecker(nodeStore.getRoot(), indexPath, workDir); + + boolean result = false; + try{ + result = checker.check(IndexConsistencyChecker.Level.BLOBS_ONLY).clean; + } catch (Exception e) { + log.warn("Error occurred while performing consistency check for {}", indexPath, e); + } + return result; + } + + private void computeAsyncIndexInfo(NodeState idxState, String indexPath, LuceneIndexInfo info) { + String asyncName = IndexUtils.getAsyncLaneName(idxState, indexPath); + if (asyncName == null) { + log.warn("No 'async' value for index definition at [{}]. Definition {}", indexPath, idxState); + return; + } + + AsyncIndexInfo asyncInfo = asyncInfoService.getInfo(asyncName); + checkNotNull(asyncInfo, "No async info found for name [%s] " + + "for index at [%s]", asyncName, indexPath); + + info.indexedUptoTime = asyncInfo.getLastIndexedTo(); + info.asyncName = asyncName; + } + + private void computeSize(NodeState idxState, LuceneIndexInfo info) throws IOException { + LuceneIndexDefinition defn = LuceneIndexDefinition.newBuilder(nodeStore.getRoot(), idxState, info.indexPath).build(); + for (String dirName : idxState.getChildNodeNames()) { + if (NodeStateUtils.isHidden(dirName) && MultiplexersLucene.isIndexDirName(dirName)) { + try (Directory dir = new OakDirectory(new ReadOnlyBuilder(idxState), dirName, defn, true)) { + info.numEntries += DirectoryUtils.getNumDocs(dir); + info.size = DirectoryUtils.dirSize(dir); + } + } + } + } + + private static void computeLastUpdatedTime(NodeState idxState, LuceneIndexInfo info) { + NodeState status = idxState.getChildNode(IndexDefinition.STATUS_NODE); + if (status.exists()){ + PropertyState updatedTime = status.getProperty(IndexDefinition.STATUS_LAST_UPDATED); + if (updatedTime != null) { + info.lastUpdatedTime = ISO8601.parse(updatedTime.getValue(Type.DATE)).getTimeInMillis(); + } + } + } + + private static void computeIndexDefinitionChange(NodeState idxState, LuceneIndexInfo info) { + NodeState storedDefn = idxState.getChildNode(INDEX_DEFINITION_NODE); + if (storedDefn.exists()) { + NodeState currentDefn = NodeStateCloner.cloneVisibleState(idxState); + if (!FilteringEqualsDiff.equals(storedDefn, currentDefn)){ + info.indexDefinitionChanged = true; + info.indexDiff = JsopDiff.diffToJsop(storedDefn, currentDefn); + } + } + } + + private static class LuceneIndexInfo implements IndexInfo { + String indexPath; + String asyncName; + long numEntries; + long size; + long indexedUptoTime; + long lastUpdatedTime; + boolean indexDefinitionChanged; + String indexDiff; + + public LuceneIndexInfo(String indexPath) { + this.indexPath = indexPath; + } + + @Override + public String getIndexPath() { + return indexPath; + } + + @Override + public String getType() { + return LuceneIndexConstants.TYPE_LUCENE; + } + + @Override + public String getAsyncLaneName() { + return asyncName; + } + + @Override + public long getLastUpdatedTime() { + return lastUpdatedTime; + } + + @Override + public long getIndexedUpToTime() { + return indexedUptoTime; + } + + @Override + public long getEstimatedEntryCount() { + return numEntries; + } + + @Override + public long getSizeInBytes() { + return size; + } + + @Override + public boolean hasIndexDefinitionChangedWithoutReindexing() { + return indexDefinitionChanged; + } + + @Override + public String getIndexDefinitionDiff() { + return indexDiff; + } + } + + static class FilteringEqualsDiff extends EqualsDiff { + private static final Set<String> IGNORED_PROP_NAMES = ImmutableSet.of( + IndexConstants.REINDEX_COUNT, + IndexConstants.REINDEX_PROPERTY_NAME + ); + public static boolean equals(NodeState before, NodeState after) { + return before.exists() == after.exists() + && after.compareAgainstBaseState(before, new FilteringEqualsDiff()); + } + + @Override + public boolean propertyChanged(PropertyState before, PropertyState after) { + if (ignoredProp(before.getName())){ + return true; + } + return false; + } + + @Override + public boolean propertyAdded(PropertyState after) { + if (ignoredProp(after.getName())){ + return true; + } + return super.propertyAdded(after); + } + + @Override + public boolean propertyDeleted(PropertyState before) { + if (ignoredProp(before.getName())){ + return true; + } + return super.propertyDeleted(before); + } + + private boolean ignoredProp(String name) { + return IGNORED_PROP_NAMES.contains(name) || NodeStateUtils.isHidden(name); + } + } +} Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java?rev=1841926&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java (added) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java Tue Sep 25 12:24:15 2018 @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.util.Collection; +import java.util.function.Predicate; + +import org.apache.jackrabbit.oak.plugins.index.search.IndexFormatVersion; +import org.apache.jackrabbit.oak.plugins.index.search.IndexLookup; +import org.apache.jackrabbit.oak.spi.query.Filter; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE; + +class LuceneIndexLookupUtil { + static final Predicate<NodeState> LUCENE_INDEX_DEFINITION_PREDICATE = + state -> TYPE_LUCENE.equals(state.getString(TYPE_PROPERTY_NAME)); + + private LuceneIndexLookupUtil() { + } + + /** + * Returns the path of the first Lucene index node which supports + * fulltext search + */ + public static String getOldFullTextIndexPath(NodeState root, Filter filter, IndexTracker tracker) { + Collection<String> indexPaths = getLuceneIndexLookup(root).collectIndexNodePaths(filter, false); + LuceneIndexNode indexNode = null; + for (String path : indexPaths) { + try { + indexNode = tracker.acquireIndexNode(path); + if (indexNode != null + && indexNode.getDefinition().isFullTextEnabled() + && indexNode.getDefinition().getVersion() == IndexFormatVersion.V1) { + return path; + } + } finally { + if (indexNode != null) { + indexNode.release(); + } + } + } + return null; + } + + public static IndexLookup getLuceneIndexLookup(NodeState root) { + return new IndexLookup(root, LUCENE_INDEX_DEFINITION_PREDICATE); + } +} Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java?rev=1841926&r1=1841925&r2=1841926&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java (original) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java Tue Sep 25 12:24:15 2018 @@ -48,7 +48,7 @@ import org.apache.jackrabbit.oak.commons import org.apache.jackrabbit.oak.json.JsopDiff; import org.apache.jackrabbit.oak.plugins.index.IndexConstants; import org.apache.jackrabbit.oak.plugins.index.IndexPathService; -import org.apache.jackrabbit.oak.plugins.index.lucene.BadIndexTracker.BadIndexInfo; +import org.apache.jackrabbit.oak.plugins.index.search.BadIndexTracker.BadIndexInfo; import org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexInfo; import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner; import org.apache.jackrabbit.oak.plugins.index.lucene.util.PathStoredFieldVisitor; @@ -56,6 +56,12 @@ import org.apache.jackrabbit.oak.plugins import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker.Level; import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker.Result; import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader; +import org.apache.jackrabbit.oak.plugins.index.search.BadIndexTracker; +import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; +import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.IndexNode; +import org.apache.jackrabbit.oak.plugins.index.search.util.NodeStateCloner; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; import org.apache.jackrabbit.oak.spi.state.NodeStore; @@ -85,7 +91,7 @@ import org.slf4j.LoggerFactory; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount; -import static org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.INDEX_DEFINITION_NODE; +import static org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newAncestorTerm; import static org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryUtils.dirSize; @@ -116,7 +122,7 @@ public class LuceneIndexMBeanImpl extend tds = new TabularDataSupport(tt); Set<String> indexes = indexTracker.getIndexNodePaths(); for (String path : indexes) { - IndexNode indexNode = null; + LuceneIndexNode indexNode = null; try { indexNode = indexTracker.acquireIndexNode(path); if (indexNode != null) { @@ -144,7 +150,7 @@ public class LuceneIndexMBeanImpl extend tds = new TabularDataSupport(tt); Set<String> indexes = indexTracker.getBadIndexTracker().getIndexPaths(); for (String path : indexes) { - BadIndexInfo info = indexTracker.getBadIndexTracker().getInfo(path); + BadIndexTracker.BadIndexInfo info = indexTracker.getBadIndexTracker().getInfo(path); if (info != null){ BadIndexStats stats = new BadIndexStats(info); tds.put(stats.toCompositeData()); @@ -184,7 +190,7 @@ public class LuceneIndexMBeanImpl extend @Override public String[] getIndexedPaths(String indexPath, int maxLevel, int maxPathCount) throws IOException { - IndexNode indexNode = null; + LuceneIndexNode indexNode = null; try { if(indexPath == null){ indexPath = "/"; @@ -195,7 +201,7 @@ public class LuceneIndexMBeanImpl extend IndexDefinition defn = indexNode.getDefinition(); if (!defn.evaluatePathRestrictions()){ String msg = String.format("Index at [%s] does not have [%s] enabled. So paths statistics cannot " + - "be determined for this index", indexPath, LuceneIndexConstants.EVALUATE_PATH_RESTRICTION); + "be determined for this index", indexPath, FulltextIndexConstants.EVALUATE_PATH_RESTRICTION); return createMsg(msg); } @@ -220,7 +226,7 @@ public class LuceneIndexMBeanImpl extend } ArrayList<String> list = new ArrayList<String>(); for (String path : indexes) { - IndexNode indexNode = null; + LuceneIndexNode indexNode = null; try { indexNode = indexTracker.acquireIndexNode(path); if (indexNode != null) { @@ -255,7 +261,7 @@ public class LuceneIndexMBeanImpl extend } ArrayList<String> list = new ArrayList<String>(); for (String path : indexes) { - IndexNode indexNode = null; + LuceneIndexNode indexNode = null; try { indexNode = indexTracker.acquireIndexNode(path); if (indexNode != null) { @@ -370,7 +376,7 @@ public class LuceneIndexMBeanImpl extend } public void dumpIndexContent(String sourcePath, String destPath) throws IOException { - IndexNode indexNode = null; + LuceneIndexNode indexNode = null; try { if(sourcePath == null){ sourcePath = "/"; @@ -681,7 +687,7 @@ public class LuceneIndexMBeanImpl extend private final String nrtIndexSizeStr; private final int numDocsNRT; - public IndexStats(String path, IndexNode indexNode) throws IOException { + public IndexStats(String path, LuceneIndexNode indexNode) throws IOException { this.path = path; numDocs = indexNode.getSearcher().getIndexReader().numDocs(); maxDoc = indexNode.getSearcher().getIndexReader().maxDoc(); @@ -755,9 +761,9 @@ public class LuceneIndexMBeanImpl extend } } - private final BadIndexInfo info; + private final BadIndexTracker.BadIndexInfo info; - public BadIndexStats(BadIndexInfo info){ + public BadIndexStats(BadIndexTracker.BadIndexInfo info){ this.info = info; } Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java?rev=1841926&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java (added) +++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java Tue Sep 25 12:24:15 2018 @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.io.IOException; +import java.util.List; + +import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader; +import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter; +import org.apache.jackrabbit.oak.plugins.index.search.IndexNode; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester; +import org.apache.lucene.store.Directory; +import org.jetbrains.annotations.Nullable; + +public interface LuceneIndexNode extends IndexNode { + + @Override + LuceneIndexDefinition getDefinition(); + + IndexSearcher getSearcher(); + + LuceneIndexStatistics getIndexStatistics(); + + List<LuceneIndexReader> getPrimaryReaders(); + + @Nullable + Directory getSuggestDirectory(); + + List<LuceneIndexReader> getNRTReaders(); + + @Nullable + AnalyzingInfixSuggester getLookup(); + + int getIndexNodeId(); + + @Nullable + LuceneIndexWriter getLocalWriter() throws IOException; + + void refreshReadersOnWriteIfRequired(); +} Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java ------------------------------------------------------------------------------ svn:eol-style = native
