This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch import-nouveau-reorg in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f233a3ba313f570571ff966f652065898dd40052 Author: Robert Newson <[email protected]> AuthorDate: Thu Dec 29 23:20:27 2022 +0000 isolate Lucene behind interface --- .tool-versions | 4 + .../apache/couchdb/nouveau/NouveauApplication.java | 26 +-- .../couchdb/nouveau/core/DocumentFactory.java | 50 ----- .../org/apache/couchdb/nouveau/core/Index.java | 179 ++++++++++++++++++ ...allelSearcherFactory.java => IndexFactory.java} | 23 +-- .../apache/couchdb/nouveau/core/IndexManager.java | 207 ++------------------- ...zerFactory.java => Lucene9AnalyzerFactory.java} | 5 +- .../apache/couchdb/nouveau/core/Lucene9Index.java | 149 +++++++++++++++ .../couchdb/nouveau/core/Lucene9IndexFactory.java | 44 +++++ ...ry.java => Lucene9ParallelSearcherFactory.java} | 2 +- .../nouveau/health/IndexManagerHealthCheck.java | 20 +- .../couchdb/nouveau/resources/AnalyzeResource.java | 6 +- .../couchdb/nouveau/resources/IndexResource.java | 28 +-- .../couchdb/nouveau/core/IndexManagerTest.java | 2 +- ...ryTest.java => Lucene9AnalyzerFactoryTest.java} | 4 +- 15 files changed, 437 insertions(+), 312 deletions(-) diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 000000000..ad8836716 --- /dev/null +++ b/.tool-versions @@ -0,0 +1,4 @@ +java zulu-11.60.19 +maven 3.8.6 +elixir 1.13.4-otp-23 +erlang 23.3.4.14 diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java index a97f9e1a3..c1e76a0dc 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java @@ -15,12 +15,13 @@ package org.apache.couchdb.nouveau; import java.util.concurrent.ExecutorService; -import org.apache.couchdb.nouveau.core.AnalyzerFactory; -import org.apache.couchdb.nouveau.core.DocumentFactory; +import org.apache.couchdb.nouveau.core.Lucene9AnalyzerFactory; +import org.apache.couchdb.nouveau.core.Lucene9IndexFactory; +import org.apache.couchdb.nouveau.core.Lucene9ParallelSearcherFactory; import org.apache.couchdb.nouveau.core.FileAlreadyExistsExceptionMapper; import org.apache.couchdb.nouveau.core.FileNotFoundExceptionMapper; +import org.apache.couchdb.nouveau.core.IndexFactory; import org.apache.couchdb.nouveau.core.IndexManager; -import org.apache.couchdb.nouveau.core.ParallelSearcherFactory; import org.apache.couchdb.nouveau.core.UpdatesOutOfOrderExceptionMapper; import org.apache.couchdb.nouveau.core.ser.LuceneModule; import org.apache.couchdb.nouveau.health.AnalyzeHealthCheck; @@ -28,6 +29,8 @@ import org.apache.couchdb.nouveau.health.IndexManagerHealthCheck; import org.apache.couchdb.nouveau.resources.AnalyzeResource; import org.apache.couchdb.nouveau.resources.IndexResource; import org.apache.couchdb.nouveau.resources.SearchResource; +import org.apache.lucene.search.SearcherManager; + import com.fasterxml.jackson.databind.ObjectMapper; import com.codahale.metrics.MetricRegistry; @@ -52,14 +55,14 @@ public class NouveauApplication extends Application<NouveauApplicationConfigurat final MetricRegistry metricsRegistry = new MetricRegistry(); environment.jersey().register(new InstrumentedResourceMethodApplicationListener(metricsRegistry)); - final DocumentFactory documentFactory = new DocumentFactory(); - final AnalyzerFactory analyzerFactory = new AnalyzerFactory(); + final Lucene9AnalyzerFactory analyzerFactory = new Lucene9AnalyzerFactory(); - final ExecutorService searchExecutor = - environment.lifecycle().executorService("nouveau-search-%d").build(); + final Lucene9ParallelSearcherFactory searcherFactory = new Lucene9ParallelSearcherFactory(); + searcherFactory.setExecutor(environment.lifecycle().executorService("nouveau-search-thread-%d").build()); - final ParallelSearcherFactory searcherFactory = new ParallelSearcherFactory(); - searcherFactory.setExecutor(searchExecutor); + final Lucene9IndexFactory indexFactory = new Lucene9IndexFactory(); + indexFactory.setAnalyzerFactory(analyzerFactory); + indexFactory.setSearcherFactory(searcherFactory); final ObjectMapper objectMapper = environment.getObjectMapper(); objectMapper.registerModule(new LuceneModule()); @@ -70,9 +73,8 @@ public class NouveauApplication extends Application<NouveauApplicationConfigurat indexManager.setMaxIndexesOpen(configuration.getMaxIndexesOpen()); indexManager.setCommitIntervalSeconds(configuration.getCommitIntervalSeconds()); indexManager.setIdleSeconds(configuration.getIdleSeconds()); - indexManager.setAnalyzerFactory(analyzerFactory); indexManager.setObjectMapper(objectMapper); - indexManager.setSearcherFactory(searcherFactory); + indexManager.setIndexFactory(indexFactory); environment.lifecycle().manage(indexManager); environment.jersey().register(new FileNotFoundExceptionMapper()); @@ -81,7 +83,7 @@ public class NouveauApplication extends Application<NouveauApplicationConfigurat final AnalyzeResource analyzeResource = new AnalyzeResource(analyzerFactory); environment.jersey().register(analyzeResource); - environment.jersey().register(new IndexResource(indexManager, documentFactory)); + environment.jersey().register(new IndexResource(indexManager)); environment.jersey().register(new SearchResource(indexManager)); // health checks diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/DocumentFactory.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/DocumentFactory.java deleted file mode 100644 index 904a215c0..000000000 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/DocumentFactory.java +++ /dev/null @@ -1,50 +0,0 @@ -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.apache.couchdb.nouveau.core; - -import java.io.IOException; - -import org.apache.couchdb.nouveau.api.DocumentUpdateRequest; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.util.BytesRef; - -public class DocumentFactory { - - public Document build(final String docId, final DocumentUpdateRequest request) throws IOException { - final Document result = new Document(); - - // id - result.add(new org.apache.lucene.document.StringField("_id", docId, Store.YES)); - result.add(new org.apache.lucene.document.SortedDocValuesField("_id", new BytesRef(docId))); - - // partition (optional) - if (request.hasPartition()) { - result.add(new org.apache.lucene.document.StringField("_partition", request.getPartition(), Store.NO)); - } - - for (IndexableField field : request.getFields()) { - // Underscore-prefix is reserved. - if (field.name().startsWith("_")) { - continue; - } - result.add(field); - } - - return result; - } - -} diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Index.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Index.java new file mode 100644 index 000000000..0f360f8a6 --- /dev/null +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Index.java @@ -0,0 +1,179 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.core; + +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.couchdb.nouveau.api.DocumentDeleteRequest; +import org.apache.couchdb.nouveau.api.DocumentUpdateRequest; +import org.apache.couchdb.nouveau.api.IndexInfo; +import org.apache.couchdb.nouveau.api.SearchRequest; +import org.apache.couchdb.nouveau.api.SearchResults; + +public abstract class Index implements Closeable { + + /* + * The close lock is to ensure there are no readers/searchers when + * we want to close the index. + */ + private ReentrantReadWriteLock closeLock = new ReentrantReadWriteLock(); + + /* + * The update lock ensures serial updates to the index. + */ + private ReentrantReadWriteLock updateLock = new ReentrantReadWriteLock(); + + private long updateSeq; + + private boolean deleteOnClose = false; + + private boolean closed = false; + + protected Index(final long updateSeq) { + this.updateSeq = updateSeq; + } + + public final IndexInfo info() throws IOException { + final long updateSeq = getUpdateSeq(); + closeLock.readLock().lock(); + try { + final int numDocs = doNumDocs(); + return new IndexInfo(updateSeq, numDocs); + } finally { + closeLock.readLock().unlock(); + } + } + + protected abstract int doNumDocs() throws IOException; + + public final void update(final String docId, final DocumentUpdateRequest request) throws IOException { + updateLock.writeLock().lock(); + try { + assertUpdateSeqIsLower(request.getSeq()); + closeLock.readLock().lock(); + try { + doUpdate(docId, request); + } finally { + closeLock.readLock().unlock(); + } + incrementUpdateSeq(request.getSeq()); + } finally { + updateLock.writeLock().unlock(); + } + } + + protected abstract void doUpdate(final String docId, final DocumentUpdateRequest request) throws IOException; + + public final void delete(final String docId, final DocumentDeleteRequest request) throws IOException { + updateLock.writeLock().lock(); + try { + assertUpdateSeqIsLower(request.getSeq()); + closeLock.readLock().lock(); + try { + doDelete(docId, request); + } finally { + closeLock.readLock().unlock(); + } + incrementUpdateSeq(request.getSeq()); + } finally { + updateLock.writeLock().unlock(); + } + } + + protected abstract void doDelete(final String docId, final DocumentDeleteRequest request) throws IOException; + + public final SearchResults search(final SearchRequest request) throws IOException { + closeLock.readLock().lock(); + try { + return doSearch(request); + } finally { + closeLock.readLock().unlock(); + } + } + + protected abstract SearchResults doSearch(final SearchRequest request) throws IOException; + + public final boolean commit() throws IOException { + final long updateSeq = getUpdateSeq(); + closeLock.readLock().lock(); + try { + return doCommit(updateSeq); + } finally { + closeLock.readLock().unlock(); + } + } + + protected abstract boolean doCommit(final long updateSeq) throws IOException; + + public final void close() throws IOException { + closeLock.writeLock().lock(); + try { + doClose(deleteOnClose); + closed = true; + } finally { + closeLock.writeLock().unlock(); + } + } + + protected abstract void doClose(final boolean deleteOnClose) throws IOException; + + public final void lock() { + closeLock.readLock().lock(); + } + + public final void unlock() { + closeLock.readLock().unlock(); + } + + public final boolean isClosed() { + return closed; + } + + public final void setDeleteOnClose(final boolean deleteOnClose) { + closeLock.writeLock().lock(); + try { + this.deleteOnClose = true; + } finally { + closeLock.writeLock().unlock(); + } + } + + protected final void assertUpdateSeqIsLower(final long updateSeq) throws UpdatesOutOfOrderException { + assert updateLock.isWriteLockedByCurrentThread(); + if (!(updateSeq > this.updateSeq)) { + throw new UpdatesOutOfOrderException(); + } + } + + protected final void incrementUpdateSeq(final long updateSeq) throws IOException { + assert updateLock.isWriteLockedByCurrentThread(); + assertUpdateSeqIsLower(updateSeq); + this.updateSeq = updateSeq; + } + + private long getUpdateSeq() { + updateLock.readLock().lock(); + try { + return this.updateSeq; + } finally { + updateLock.readLock().unlock(); + } + } + +} \ No newline at end of file diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexFactory.java similarity index 50% copy from java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java copy to java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexFactory.java index bd31801fd..8d728c080 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexFactory.java @@ -14,27 +14,12 @@ package org.apache.couchdb.nouveau.core; import java.io.IOException; -import java.util.concurrent.Executor; +import java.nio.file.Path; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.SearcherFactory; +import org.apache.couchdb.nouveau.api.IndexDefinition; -public class ParallelSearcherFactory extends SearcherFactory { +public interface IndexFactory { - private Executor executor; - - public Executor getExecutor() { - return executor; - } - - public void setExecutor(Executor executor) { - this.executor = executor; - } - - @Override - public IndexSearcher newSearcher(final IndexReader reader, final IndexReader previousReader) throws IOException { - return new IndexSearcher(reader, executor); - } + Index open(final Path path, final IndexDefinition indexDefinition) throws IOException; } diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java index dd0f1f2e0..b00fe4164 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java @@ -18,13 +18,7 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; -import java.util.Collections; -import java.util.Map; import java.util.concurrent.CompletionException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Stream; import javax.validation.constraints.Min; @@ -34,21 +28,13 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response.Status; import org.apache.couchdb.nouveau.api.IndexDefinition; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.misc.store.DirectIODirectory; -import org.apache.lucene.search.SearcherFactory; -import org.apache.lucene.search.SearcherManager; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.util.IOUtils; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.caffeine.MetricsStatsCounter; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.benmanes.caffeine.cache.CacheLoader; @@ -58,135 +44,15 @@ import com.github.benmanes.caffeine.cache.RemovalCause; import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Scheduler; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.caffeine.MetricsStatsCounter; - import io.dropwizard.lifecycle.Managed; public class IndexManager implements Managed { private static final int RETRY_LIMIT = 500; private static final int RETRY_SLEEP_MS = 5; + private static final String DEFAULT_FIELD = "default"; private static final Logger LOGGER = LoggerFactory.getLogger(IndexManager.class); - public class Index { - private static final String DEFAULT_FIELD = "default"; - private final String name; - private IndexWriter writer; - private SearcherManager searcherManager; - private Analyzer analyzer; - private final AtomicBoolean deleteOnClose = new AtomicBoolean(); - private final AtomicLong updateSeq = new AtomicLong(); - - // The write lock is to ensure there are no readers/searchers when - // we want to close the index. - private ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(); - private Lock rl = rwl.readLock(); - private Lock wl = rwl.writeLock(); - - private Index( - String name, - IndexWriter writer, - SearcherManager searcherManager, - Analyzer analyzer, - long updateSeq) { - this.name = name; - this.writer = writer; - this.searcherManager = searcherManager; - this.analyzer = analyzer; - this.updateSeq.set(updateSeq); - } - - public String getName() { - return name; - } - - public IndexWriter getWriter() { - return writer; - } - - public SearcherManager getSearcherManager() { - return searcherManager; - } - - public QueryParser getQueryParser() { - return new NouveauQueryParser(DEFAULT_FIELD, analyzer); - } - - public boolean commit() throws IOException { - rl.lock(); - try { - writer.setLiveCommitData(generateCommitData().entrySet()); - return writer.commit() != -1; - } finally { - rl.unlock(); - } - } - - public long getUpdateSeq() throws IOException { - return updateSeq.get(); - } - - public void incrementUpdateSeq(final long updateSeq) throws IOException { - final long newSeq = this.updateSeq.accumulateAndGet(updateSeq, (a, b) -> Math.max(a, b)); - if (newSeq != updateSeq) { - throw new UpdatesOutOfOrderException(); - } - } - - public void close() throws IOException { - wl.lock(); - try { - if (writer == null) { - // Already closed. - return; - } - - // Close searcher manager - if (searcherManager != null) { - try { - searcherManager.close(); - } catch (IOException e) { - LOGGER.info(this + " threw exception when closing searcherManager.", e); - } finally { - searcherManager = null; - } - } - - if (deleteOnClose.get()) { - try { - // No need to commit in this case. - writer.rollback(); - } catch (IOException e) { - LOGGER.info(this + " threw exception when rolling back writer.", e); - } finally { - writer = null; - } - IOUtils.rm(indexRootPath(name)); - } else { - try { - writer.setLiveCommitData(generateCommitData().entrySet()); - writer.close(); - LOGGER.info("{} closed.", this); - } finally { - writer = null; - } - } - } finally { - wl.unlock(); - } - } - - private Map<String, String> generateCommitData() { - return Collections.singletonMap("update_seq", Long.toString(updateSeq.get())); - } - - @Override - public String toString() { - return "Index [name=" + name + "]"; - } - } - private class IndexLoader implements CacheLoader<String, Index> { @Override @@ -237,12 +103,12 @@ public class IndexManager implements Managed { private Path rootDir; @NotNull - private AnalyzerFactory analyzerFactory; + private Lucene9AnalyzerFactory analyzerFactory; @NotNull private ObjectMapper objectMapper; - private SearcherFactory searcherFactory; + private IndexFactory indexFactory; private MetricRegistry metricRegistry; @@ -253,11 +119,11 @@ public class IndexManager implements Managed { final Index result = getFromCache(name); // Check if we're in the middle of closing. - result.rl.lock(); - if (result.writer != null) { + result.lock(); + if (!result.isClosed()) { return result; } - result.rl.unlock(); + result.unlock(); // Retry after a short sleep. try { @@ -271,7 +137,7 @@ public class IndexManager implements Managed { } public void release(final Index index) throws IOException { - index.rl.unlock(); + index.unlock(); } public void create(final String name, IndexDefinition indexDefinition) throws IOException { @@ -301,7 +167,7 @@ public class IndexManager implements Managed { private void deleteIndex(final String name) throws IOException { final Index index = acquire(name); try { - index.deleteOnClose.set(true); + index.setDeleteOnClose(true); cache.invalidate(name); } finally { release(index); @@ -344,7 +210,7 @@ public class IndexManager implements Managed { this.rootDir = rootDir; } - public void setAnalyzerFactory(final AnalyzerFactory analyzerFactory) { + public void setAnalyzerFactory(final Lucene9AnalyzerFactory analyzerFactory) { this.analyzerFactory = analyzerFactory; } @@ -352,8 +218,8 @@ public class IndexManager implements Managed { this.objectMapper = objectMapper; } - public void setSearcherFactory(final SearcherFactory searcherFactory) { - this.searcherFactory = searcherFactory; + public void setIndexFactory(final IndexFactory indexFactory) { + this.indexFactory = indexFactory; } public void setMetricRegistry(final MetricRegistry metricRegistry) { @@ -405,48 +271,9 @@ public class IndexManager implements Managed { } private Index openExistingIndex(final String name) throws IOException { - final IndexDefinition indexDefinition = objectMapper.readValue(indexDefinitionPath(name).toFile(), IndexDefinition.class); - final Analyzer analyzer = analyzerFactory.fromDefinition(indexDefinition); final Path path = indexPath(name); - final Directory dir = directory(path); - final IndexWriter writer = newWriter(dir, analyzer); - final SearcherManager searcherManager = new SearcherManager(writer, searcherFactory); - final long updateSeq = getUpdateSeq(writer); - return new Index(name, writer, searcherManager, analyzer, updateSeq); - } - - private long getUpdateSeq(final IndexWriter writer) throws IOException { - final Iterable<Map.Entry<String, String>> commitData = writer.getLiveCommitData(); - if (commitData == null) { - return 0L; - } - for (Map.Entry<String, String> entry : commitData) { - if (entry.getKey().equals("update_seq")) { - return Long.parseLong(entry.getValue()); - } - } - return 0L; - } - - private IndexWriter newWriter(final Directory dir, final Analyzer analyzer) throws IOException { - LockObtainFailedException exceptionThrown = null; - for (int i = 0; i < RETRY_LIMIT; i++) { - try { - final IndexWriterConfig config = new IndexWriterConfig(analyzer); - config.setCommitOnClose(true); - config.setUseCompoundFile(false); - return new IndexWriter(dir, config); - } catch (LockObtainFailedException e) { - exceptionThrown = e; - try { - Thread.sleep(RETRY_SLEEP_MS); - } catch (InterruptedException e1) { - Thread.interrupted(); - break; - } - } - } - throw exceptionThrown; + final IndexDefinition indexDefinition = objectMapper.readValue(indexDefinitionPath(name).toFile(), IndexDefinition.class); + return indexFactory.open(path, indexDefinition); } private boolean isIndex(final Path path) { @@ -470,8 +297,4 @@ public class IndexManager implements Managed { Status.BAD_REQUEST); } - private Directory directory(final Path path) throws IOException { - return new DirectIODirectory(FSDirectory.open(path)); - } - } diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/AnalyzerFactory.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactory.java similarity index 98% rename from java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/AnalyzerFactory.java rename to java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactory.java index 0ad6c0311..5544af267 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/AnalyzerFactory.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactory.java @@ -62,7 +62,10 @@ import org.apache.lucene.analysis.sv.SwedishAnalyzer; import org.apache.lucene.analysis.th.ThaiAnalyzer; import org.apache.lucene.analysis.tr.TurkishAnalyzer; -public class AnalyzerFactory { +public final class Lucene9AnalyzerFactory { + + public Lucene9AnalyzerFactory() { + } public Analyzer fromDefinition(final IndexDefinition indexDefinition) { final Analyzer defaultAnalyzer = newAnalyzer(indexDefinition.getDefaultAnalyzer()); diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9Index.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9Index.java new file mode 100644 index 000000000..246fcf7ae --- /dev/null +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9Index.java @@ -0,0 +1,149 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.core; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; + +import org.apache.couchdb.nouveau.api.DocumentDeleteRequest; +import org.apache.couchdb.nouveau.api.DocumentUpdateRequest; +import org.apache.couchdb.nouveau.api.SearchRequest; +import org.apache.couchdb.nouveau.api.SearchResults; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.misc.store.DirectIODirectory; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SearcherFactory; +import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; + +class Lucene9Index extends Index { + + private final IndexWriter writer; + private final SearcherManager searcherManager; + + static Index open(final Path path, final Analyzer analyzer, final SearcherFactory searcherFactory) throws IOException { + final Directory dir = new DirectIODirectory(FSDirectory.open(path)); + final IndexWriterConfig config = new IndexWriterConfig(analyzer); + config.setCommitOnClose(true); + config.setUseCompoundFile(false); + final IndexWriter writer = new IndexWriter(dir, config); + final long updateSeq = getUpdateSeq(writer); + final SearcherManager searcherManager = new SearcherManager(dir, searcherFactory); + return new Lucene9Index(writer, updateSeq, searcherManager); + } + + private Lucene9Index(final IndexWriter writer, final long updateSeq, final SearcherManager searcherManager) { + super(updateSeq); + this.writer = writer; + this.searcherManager = searcherManager; + } + + @Override + public int doNumDocs() throws IOException { + return writer.getDocStats().numDocs; + } + + @Override + public void doUpdate(final String docId, final DocumentUpdateRequest request) throws IOException { + final Term docIdTerm = docIdTerm(docId); + final Document doc = toDocument(docId, request); + writer.updateDocument(docIdTerm, doc); + } + + @Override + public void doDelete(final String docId, final DocumentDeleteRequest request) throws IOException { + final Query query = docIdQuery(docId); + writer.deleteDocuments(query); + } + + @Override + public SearchResults doSearch(SearchRequest request) throws IOException { + return null; + } + + @Override + public boolean doCommit(final long updateSeq) throws IOException { + writer.setLiveCommitData(Collections.singletonMap("update_seq", Long.toString(updateSeq)).entrySet()); + return writer.commit() != -1; + } + + @Override + public void doClose(final boolean deleteOnClose) throws IOException { + if (deleteOnClose) { + // No need to commit in this case. + writer.rollback(); + final Directory dir = writer.getDirectory(); + for (final String name : dir.listAll()) { + dir.deleteFile(name); + } + } + writer.close(); + } + + private static Document toDocument(final String docId, final DocumentUpdateRequest request) throws IOException { + final Document result = new Document(); + + // id + result.add(new org.apache.lucene.document.StringField("_id", docId, Store.YES)); + result.add(new org.apache.lucene.document.SortedDocValuesField("_id", new BytesRef(docId))); + + // partition (optional) + if (request.hasPartition()) { + result.add(new org.apache.lucene.document.StringField("_partition", request.getPartition(), Store.NO)); + } + + for (IndexableField field : request.getFields()) { + // Underscore-prefix is reserved. + if (field.name().startsWith("_")) { + continue; + } + result.add(field); + } + + return result; + } + + private static Query docIdQuery(final String docId) { + return new TermQuery(docIdTerm(docId)); + } + + private static Term docIdTerm(final String docId) { + return new Term("_id", docId); + } + + private static long getUpdateSeq(final IndexWriter writer) throws IOException { + final Iterable<Map.Entry<String, String>> commitData = writer.getLiveCommitData(); + if (commitData == null) { + return 0L; + } + for (Map.Entry<String, String> entry : commitData) { + if (entry.getKey().equals("update_seq")) { + return Long.parseLong(entry.getValue()); + } + } + return 0L; + } + +} diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9IndexFactory.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9IndexFactory.java new file mode 100644 index 000000000..1d0a4b263 --- /dev/null +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9IndexFactory.java @@ -0,0 +1,44 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.core; + +import java.io.IOException; +import java.nio.file.Path; + +import org.apache.couchdb.nouveau.api.IndexDefinition; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.search.SearcherFactory; +import org.apache.lucene.search.SearcherManager; + +public class Lucene9IndexFactory implements IndexFactory { + + private Lucene9AnalyzerFactory analyzerFactory; + + private SearcherFactory searcherFactory; + + public void setAnalyzerFactory(final Lucene9AnalyzerFactory analyzerFactory) { + this.analyzerFactory = analyzerFactory; + } + + public void setSearcherFactory(final SearcherFactory searcherFactory) { + this.searcherFactory = searcherFactory; + } + + @Override + public Index open(Path path, final IndexDefinition indexDefinition) throws IOException { + final Analyzer analyzer = analyzerFactory.fromDefinition(indexDefinition); + return Lucene9Index.open(path, analyzer, searcherFactory); + } + +} diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9ParallelSearcherFactory.java similarity index 94% rename from java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java rename to java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9ParallelSearcherFactory.java index bd31801fd..7021f2997 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9ParallelSearcherFactory.java @@ -20,7 +20,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SearcherFactory; -public class ParallelSearcherFactory extends SearcherFactory { +public class Lucene9ParallelSearcherFactory extends SearcherFactory { private Executor executor; diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/health/IndexManagerHealthCheck.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/health/IndexManagerHealthCheck.java index b2653db38..49f68890e 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/health/IndexManagerHealthCheck.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/health/IndexManagerHealthCheck.java @@ -15,10 +15,12 @@ package org.apache.couchdb.nouveau.health; import java.io.IOException; +import org.apache.couchdb.nouveau.api.DocumentUpdateRequest; import org.apache.couchdb.nouveau.api.IndexDefinition; import static org.apache.couchdb.nouveau.api.LuceneVersion.*; + +import org.apache.couchdb.nouveau.core.Index; import org.apache.couchdb.nouveau.core.IndexManager; -import org.apache.couchdb.nouveau.core.IndexManager.Index; import com.codahale.metrics.health.HealthCheck; import org.apache.lucene.document.Document; @@ -44,14 +46,14 @@ public class IndexManagerHealthCheck extends HealthCheck { indexManager.create(name, new IndexDefinition(LUCENE_9, "standard", null)); final Index index = indexManager.acquire(name); try { - final IndexWriter writer = index.getWriter(); - try { - writer.addDocument(new Document()); - writer.commit(); - return Result.healthy(); - } finally { - indexManager.deleteAll(name); - } + final DocumentUpdateRequest request = new DocumentUpdateRequest(); + index.update("foo", request); + index.commit(); + index.setDeleteOnClose(true); + index.close(); + return Result.healthy(); + } catch (final IOException e) { + return Result.unhealthy(e); } finally { indexManager.release(index); } diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java index 60e8c8ca3..c495cf5fa 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java @@ -29,7 +29,7 @@ import javax.ws.rs.core.Response.Status; import org.apache.couchdb.nouveau.api.AnalyzeRequest; import org.apache.couchdb.nouveau.api.AnalyzeResponse; -import org.apache.couchdb.nouveau.core.AnalyzerFactory; +import org.apache.couchdb.nouveau.core.Lucene9AnalyzerFactory; import com.codahale.metrics.annotation.Timed; import org.apache.lucene.analysis.Analyzer; @@ -41,9 +41,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @Produces(MediaType.APPLICATION_JSON) public class AnalyzeResource { - private final AnalyzerFactory analyzerFactory; + private final Lucene9AnalyzerFactory analyzerFactory; - public AnalyzeResource(AnalyzerFactory analyzerFactory) { + public AnalyzeResource(Lucene9AnalyzerFactory analyzerFactory) { this.analyzerFactory = analyzerFactory; } diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java index cd10226db..ca7ee15c0 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java @@ -30,15 +30,10 @@ import org.apache.couchdb.nouveau.api.DocumentDeleteRequest; import org.apache.couchdb.nouveau.api.DocumentUpdateRequest; import org.apache.couchdb.nouveau.api.IndexDefinition; import org.apache.couchdb.nouveau.api.IndexInfo; -import org.apache.couchdb.nouveau.core.DocumentFactory; +import org.apache.couchdb.nouveau.core.Index; import org.apache.couchdb.nouveau.core.IndexManager; -import org.apache.couchdb.nouveau.core.IndexManager.Index; -import com.codahale.metrics.annotation.Timed; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.TermQuery; +import com.codahale.metrics.annotation.Timed; @Path("/index/{name}") @Consumes(MediaType.APPLICATION_JSON) @@ -46,26 +41,20 @@ import org.apache.lucene.search.TermQuery; public class IndexResource { private final IndexManager indexManager; - private final DocumentFactory documentFactory; - public IndexResource(final IndexManager indexManager, final DocumentFactory documentFactory) { + public IndexResource(final IndexManager indexManager) { this.indexManager = indexManager; - this.documentFactory = documentFactory; } @GET @SuppressWarnings("resource") public IndexInfo indexInfo(@PathParam("name") String name) throws IOException { - final long updateSeq; - final int numDocs; final Index index = indexManager.acquire(name); try { - updateSeq = index.getUpdateSeq(); - numDocs = index.getWriter().getDocStats().numDocs; + return index.info(); } finally { indexManager.release(index); } - return new IndexInfo(updateSeq, numDocs); } @DELETE @@ -84,9 +73,7 @@ public class IndexResource { public void deleteDoc(@PathParam("name") String name, @PathParam("docId") String docId, @NotNull @Valid final DocumentDeleteRequest request) throws IOException { final Index index = indexManager.acquire(name); try { - final IndexWriter writer = index.getWriter(); - writer.deleteDocuments(new TermQuery(new Term("_id", docId))); - index.incrementUpdateSeq(request.getSeq()); + index.delete(docId, request); } finally { indexManager.release(index); } @@ -98,10 +85,7 @@ public class IndexResource { public void updateDoc(@PathParam("name") String name, @PathParam("docId") String docId, @NotNull @Valid final DocumentUpdateRequest request) throws IOException { final Index index = indexManager.acquire(name); try { - final IndexWriter writer = index.getWriter(); - final Document doc = documentFactory.build(docId, request); - writer.updateDocument(new Term("_id", docId), doc); - index.incrementUpdateSeq(request.getSeq()); + index.update(docId, request); } finally { indexManager.release(index); } diff --git a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java index 64081d5c0..4487b69e3 100644 --- a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java +++ b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java @@ -42,7 +42,7 @@ public class IndexManagerTest { public void setup() throws Exception { manager = new IndexManager(); manager.setMetricRegistry(new MetricRegistry()); - manager.setAnalyzerFactory(new AnalyzerFactory()); + manager.setAnalyzerFactory(new Lucene9AnalyzerFactory()); manager.setCommitIntervalSeconds(5); manager.setObjectMapper(new ObjectMapper()); manager.setRootDir(tempDir); diff --git a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/AnalyzerFactoryTest.java b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactoryTest.java similarity index 98% rename from java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/AnalyzerFactoryTest.java rename to java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactoryTest.java index 43bdb4e14..0adf18d45 100644 --- a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/AnalyzerFactoryTest.java +++ b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactoryTest.java @@ -56,7 +56,7 @@ import org.apache.lucene.analysis.th.ThaiAnalyzer; import org.apache.lucene.analysis.tr.TurkishAnalyzer; import org.junit.jupiter.api.Test; -public class AnalyzerFactoryTest { +public class Lucene9AnalyzerFactoryTest { @Test public void testkeyword() throws Exception { @@ -249,7 +249,7 @@ public class AnalyzerFactoryTest { } private void assertAnalyzer(final String name, final Class<? extends Analyzer> clazz) throws Exception { - final AnalyzerFactory factory = new AnalyzerFactory(); + final Lucene9AnalyzerFactory factory = new Lucene9AnalyzerFactory(); assertThat(factory.newAnalyzer(name)).isInstanceOf(clazz); }
