amit-jain commented on code in PR #2817: URL: https://github.com/apache/jackrabbit-oak/pull/2817#discussion_r3841116442
########## oak-search-lucene-ng/src/main/java/org/apache/jackrabbit/oak/plugins/index/luceneNg/LuceneNgIndexTracker.java: ########## @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.luceneNg; + +import org.apache.jackrabbit.oak.plugins.index.luceneNg.internal.LuceneNgIndexNode; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Tracks Lucene 9 indexes and provides access to index nodes. + * Scans the repository for lucene9 type indexes and maintains a cache. + */ +public class LuceneNgIndexTracker { Review Comment: Again this should extend from the shared [FulltextIndexTracker](https://github.com/apache/jackrabbit-oak/blob/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndexTracker.java#L55) This is critical as the shared tracker has a lot of concurrency issues which have been resolved (actually LuceneIndexTracker does not extend this even though the code is almost the same) * https://github.com/bhabegger/jackrabbit-oak/blob/c335365941593ac2651d82f07c07443adefd9e6e/oak-search-lucene-ng/src/main/java/org/apache/jackrabbit/oak/plugins/index/luceneNg/LuceneNgIndexTracker.java#L118 - uses index state which could be expensive The code can reuse the metadata written by the FulltextIndexEditor - https://github.com/bhabegger/jackrabbit-oak/blob/c335365941593ac2651d82f07c07443adefd9e6e/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndexTracker.java#L133 * Nested index changes discovery (https://github.com/bhabegger/jackrabbit-oak/blob/c335365941593ac2651d82f07c07443adefd9e6e/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndexTracker.java#L222-L243) ########## oak-search-lucene-ng/src/main/java/org/apache/jackrabbit/oak/plugins/index/luceneNg/LuceneNgIndexEditor.java: ########## @@ -0,0 +1,809 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.luceneNg; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; +import org.apache.jackrabbit.oak.plugins.index.luceneNg.directory.OakDirectory; +import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.IndexingRule; +import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition; +import org.apache.jackrabbit.oak.spi.commit.Editor; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.util.ISO8601; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleDocValuesField; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.facet.FacetsConfig; +import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetField; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.util.BytesRef; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.jcr.PropertyType; +import java.io.IOException; + +/** + * IndexEditor for Lucene 9. + * + * <p>Only indexes properties that are explicitly declared in the index definition's + * {@code indexRules}. This mirrors the behaviour of the legacy {@code oak-lucene} + * module and avoids the Lucene doc-values type-consistency constraint: since the + * declared type for a property is fixed at index-definition time, every document + * that contributes a doc-values field for that property will use the same type.</p> + */ +public class LuceneNgIndexEditor implements Editor { Review Comment: This should extend from [FulltextIndexEditor](https://github.com/apache/jackrabbit-oak/blob/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java) as is done for [LuceneIndexEditor](https://github.com/apache/jackrabbit-oak/blob/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java#L30) The main things that come along with the implementation is: * Aggregation support * and some writing some status markers which the shared Tracker extension can then reuse * any fixes that go into that, looks like https://github.com/apache/jackrabbit-oak/blob/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java#L305 is also missing from here ########## oak-search-lucene-ng/src/main/java/org/apache/jackrabbit/oak/plugins/index/luceneNg/internal/IndexSearcherHolder.java: ########## @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.luceneNg.internal; + +import org.apache.jackrabbit.oak.plugins.index.luceneNg.LuceneNgIndexStorage; +import org.apache.jackrabbit.oak.plugins.index.luceneNg.directory.OakDirectory; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.search.IndexSearcher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Manages IndexSearcher lifecycle for a Lucene 9 index. + * Opens the index from the {@link LuceneNgIndexStorage} node state passed in (typically the + * {@link LuceneNgIndexStorage#STORAGE_NODE_NAME} child under the index definition). + */ +public class IndexSearcherHolder implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(IndexSearcherHolder.class); + + private final String indexName; + private DirectoryReader reader; + private IndexSearcher searcher; + private OakDirectory directory; + private final ConcurrentMap<String, DefaultSortedSetDocValuesReaderState> facetStateCache = + new ConcurrentHashMap<>(); + + /** + * @param storageState {@link LuceneNgIndexStorage#storageState(NodeState)} for the index definition + * @param indexName the index name, used only for logging/error messages + */ + + public IndexSearcherHolder(NodeState storageState, String indexName) throws IOException { + this.indexName = indexName; + this.directory = new OakDirectory(storageState.builder(), indexName, true); Review Comment: NRT handling is an important part ans is also missing i think -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
