This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch import-nouveau-reorg in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 92a20a7238310d098e1f48da4ab71c2c7f7baf59 Author: Robert Newson <[email protected]> AuthorDate: Thu Dec 29 23:20:27 2022 +0000 isolate Lucene behind interface --- .tool-versions | 4 + Makefile | 4 +- .../apache/couchdb/nouveau/NouveauApplication.java | 28 ++- .../couchdb/nouveau/core/DocumentFactory.java | 50 ----- .../org/apache/couchdb/nouveau/core/Index.java | 176 ++++++++++++++++++ ...allelSearcherFactory.java => IndexFactory.java} | 23 +-- .../apache/couchdb/nouveau/core/IndexManager.java | 206 ++------------------- ...zerFactory.java => Lucene9AnalyzerFactory.java} | 5 +- .../SearchResource.java => core/Lucene9Index.java} | 194 ++++++++++++++----- .../couchdb/nouveau/core/Lucene9IndexFactory.java | 43 +++++ ...ry.java => Lucene9ParallelSearcherFactory.java} | 2 +- .../nouveau/health/IndexManagerHealthCheck.java | 27 +-- .../couchdb/nouveau/resources/AnalyzeResource.java | 6 +- .../couchdb/nouveau/resources/IndexResource.java | 28 +-- .../couchdb/nouveau/resources/SearchResource.java | 196 +------------------- .../apache/couchdb/nouveau/IntegrationTest.java | 10 +- .../couchdb/nouveau/api/SearchRequestTest.java | 4 +- .../couchdb/nouveau/core/IndexManagerTest.java | 2 +- ...ryTest.java => Lucene9AnalyzerFactoryTest.java} | 4 +- 19 files changed, 443 insertions(+), 569 deletions(-) diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 000000000..ad8836716 --- /dev/null +++ b/.tool-versions @@ -0,0 +1,4 @@ +java zulu-11.60.19 +maven 3.8.6 +elixir 1.13.4-otp-23 +erlang 23.3.4.14 diff --git a/Makefile b/Makefile index 19c1af066..5c77b2868 100644 --- a/Makefile +++ b/Makefile @@ -516,7 +516,7 @@ derived: .PHONY: nouveau nouveau: - @cd java/nouveau && mvn + @cd java/nouveau && mvn test .PHONY: nouveau-clean nouveau-clean: @@ -524,4 +524,4 @@ nouveau-clean: .PHONY: nouveau-start nouveau-start: nouveau - @cd java/nouveau && mvn exec:exec -Dexec.executable="java" + @cd java/nouveau/server && mvn exec:exec -Dexec.executable="java" diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java index a97f9e1a3..215483d84 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java @@ -13,14 +13,12 @@ package org.apache.couchdb.nouveau; -import java.util.concurrent.ExecutorService; - -import org.apache.couchdb.nouveau.core.AnalyzerFactory; -import org.apache.couchdb.nouveau.core.DocumentFactory; import org.apache.couchdb.nouveau.core.FileAlreadyExistsExceptionMapper; import org.apache.couchdb.nouveau.core.FileNotFoundExceptionMapper; import org.apache.couchdb.nouveau.core.IndexManager; -import org.apache.couchdb.nouveau.core.ParallelSearcherFactory; +import org.apache.couchdb.nouveau.core.Lucene9AnalyzerFactory; +import org.apache.couchdb.nouveau.core.Lucene9IndexFactory; +import org.apache.couchdb.nouveau.core.Lucene9ParallelSearcherFactory; import org.apache.couchdb.nouveau.core.UpdatesOutOfOrderExceptionMapper; import org.apache.couchdb.nouveau.core.ser.LuceneModule; import org.apache.couchdb.nouveau.health.AnalyzeHealthCheck; @@ -28,10 +26,10 @@ import org.apache.couchdb.nouveau.health.IndexManagerHealthCheck; import org.apache.couchdb.nouveau.resources.AnalyzeResource; import org.apache.couchdb.nouveau.resources.IndexResource; import org.apache.couchdb.nouveau.resources.SearchResource; -import com.fasterxml.jackson.databind.ObjectMapper; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.jersey2.InstrumentedResourceMethodApplicationListener; +import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.Application; import io.dropwizard.setup.Environment; @@ -52,14 +50,14 @@ public class NouveauApplication extends Application<NouveauApplicationConfigurat final MetricRegistry metricsRegistry = new MetricRegistry(); environment.jersey().register(new InstrumentedResourceMethodApplicationListener(metricsRegistry)); - final DocumentFactory documentFactory = new DocumentFactory(); - final AnalyzerFactory analyzerFactory = new AnalyzerFactory(); + final Lucene9AnalyzerFactory analyzerFactory = new Lucene9AnalyzerFactory(); - final ExecutorService searchExecutor = - environment.lifecycle().executorService("nouveau-search-%d").build(); + final Lucene9ParallelSearcherFactory searcherFactory = new Lucene9ParallelSearcherFactory(); + searcherFactory.setExecutor(environment.lifecycle().executorService("nouveau-search-thread-%d").build()); - final ParallelSearcherFactory searcherFactory = new ParallelSearcherFactory(); - searcherFactory.setExecutor(searchExecutor); + final Lucene9IndexFactory indexFactory = new Lucene9IndexFactory(); + indexFactory.setAnalyzerFactory(analyzerFactory); + indexFactory.setSearcherFactory(searcherFactory); final ObjectMapper objectMapper = environment.getObjectMapper(); objectMapper.registerModule(new LuceneModule()); @@ -70,9 +68,9 @@ public class NouveauApplication extends Application<NouveauApplicationConfigurat indexManager.setMaxIndexesOpen(configuration.getMaxIndexesOpen()); indexManager.setCommitIntervalSeconds(configuration.getCommitIntervalSeconds()); indexManager.setIdleSeconds(configuration.getIdleSeconds()); - indexManager.setAnalyzerFactory(analyzerFactory); indexManager.setObjectMapper(objectMapper); - indexManager.setSearcherFactory(searcherFactory); + indexManager.setAnalyzerFactory(analyzerFactory); + indexManager.setIndexFactory(indexFactory); environment.lifecycle().manage(indexManager); environment.jersey().register(new FileNotFoundExceptionMapper()); @@ -81,7 +79,7 @@ public class NouveauApplication extends Application<NouveauApplicationConfigurat final AnalyzeResource analyzeResource = new AnalyzeResource(analyzerFactory); environment.jersey().register(analyzeResource); - environment.jersey().register(new IndexResource(indexManager, documentFactory)); + environment.jersey().register(new IndexResource(indexManager)); environment.jersey().register(new SearchResource(indexManager)); // health checks diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/DocumentFactory.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/DocumentFactory.java deleted file mode 100644 index 904a215c0..000000000 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/DocumentFactory.java +++ /dev/null @@ -1,50 +0,0 @@ -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.apache.couchdb.nouveau.core; - -import java.io.IOException; - -import org.apache.couchdb.nouveau.api.DocumentUpdateRequest; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.util.BytesRef; - -public class DocumentFactory { - - public Document build(final String docId, final DocumentUpdateRequest request) throws IOException { - final Document result = new Document(); - - // id - result.add(new org.apache.lucene.document.StringField("_id", docId, Store.YES)); - result.add(new org.apache.lucene.document.SortedDocValuesField("_id", new BytesRef(docId))); - - // partition (optional) - if (request.hasPartition()) { - result.add(new org.apache.lucene.document.StringField("_partition", request.getPartition(), Store.NO)); - } - - for (IndexableField field : request.getFields()) { - // Underscore-prefix is reserved. - if (field.name().startsWith("_")) { - continue; - } - result.add(field); - } - - return result; - } - -} diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Index.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Index.java new file mode 100644 index 000000000..739768913 --- /dev/null +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Index.java @@ -0,0 +1,176 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.core; + +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.couchdb.nouveau.api.DocumentDeleteRequest; +import org.apache.couchdb.nouveau.api.DocumentUpdateRequest; +import org.apache.couchdb.nouveau.api.IndexInfo; +import org.apache.couchdb.nouveau.api.SearchRequest; +import org.apache.couchdb.nouveau.api.SearchResults; + +public abstract class Index implements Closeable { + + /* + * The close lock is to ensure there are no readers/searchers when + * we want to close the index. + */ + private ReentrantReadWriteLock closeLock = new ReentrantReadWriteLock(); + + /* + * The update lock ensures serial updates to the index. + */ + private ReentrantReadWriteLock updateLock = new ReentrantReadWriteLock(); + + private long updateSeq; + + private boolean deleteOnClose = false; + + private boolean closed = false; + + protected Index(final long updateSeq) { + this.updateSeq = updateSeq; + } + + public final IndexInfo info() throws IOException { + final long updateSeq = getUpdateSeq(); + closeLock.readLock().lock(); + try { + final int numDocs = doNumDocs(); + return new IndexInfo(updateSeq, numDocs); + } finally { + closeLock.readLock().unlock(); + } + } + + protected abstract int doNumDocs() throws IOException; + + public final void update(final String docId, final DocumentUpdateRequest request) throws IOException { + updateLock.writeLock().lock(); + try { + assertUpdateSeqIsLower(request.getSeq()); + closeLock.readLock().lock(); + try { + doUpdate(docId, request); + } finally { + closeLock.readLock().unlock(); + } + incrementUpdateSeq(request.getSeq()); + } finally { + updateLock.writeLock().unlock(); + } + } + + protected abstract void doUpdate(final String docId, final DocumentUpdateRequest request) throws IOException; + + public final void delete(final String docId, final DocumentDeleteRequest request) throws IOException { + updateLock.writeLock().lock(); + try { + assertUpdateSeqIsLower(request.getSeq()); + closeLock.readLock().lock(); + try { + doDelete(docId, request); + } finally { + closeLock.readLock().unlock(); + } + incrementUpdateSeq(request.getSeq()); + } finally { + updateLock.writeLock().unlock(); + } + } + + protected abstract void doDelete(final String docId, final DocumentDeleteRequest request) throws IOException; + + public final SearchResults search(final SearchRequest request) throws IOException, QueryParserException { + closeLock.readLock().lock(); + try { + return doSearch(request); + } finally { + closeLock.readLock().unlock(); + } + } + + protected abstract SearchResults doSearch(final SearchRequest request) throws IOException, QueryParserException; + + public final boolean commit() throws IOException { + final long updateSeq = getUpdateSeq(); + closeLock.readLock().lock(); + try { + return doCommit(updateSeq); + } finally { + closeLock.readLock().unlock(); + } + } + + protected abstract boolean doCommit(final long updateSeq) throws IOException; + + public final void close() throws IOException { + closeLock.writeLock().lock(); + try { + doClose(deleteOnClose); + closed = true; + } finally { + closeLock.writeLock().unlock(); + } + } + + protected abstract void doClose(final boolean deleteOnClose) throws IOException; + + public final void lock() { + closeLock.readLock().lock(); + } + + public final void unlock() { + closeLock.readLock().unlock(); + } + + public final boolean isClosed() { + return closed; + } + + public final void setDeleteOnClose(final boolean deleteOnClose) { + closeLock.writeLock().lock(); + try { + this.deleteOnClose = true; + } finally { + closeLock.writeLock().unlock(); + } + } + + protected final void assertUpdateSeqIsLower(final long updateSeq) throws UpdatesOutOfOrderException { + assert updateLock.isWriteLockedByCurrentThread(); + if (!(updateSeq > this.updateSeq)) { + throw new UpdatesOutOfOrderException(); + } + } + + protected final void incrementUpdateSeq(final long updateSeq) throws IOException { + assert updateLock.isWriteLockedByCurrentThread(); + assertUpdateSeqIsLower(updateSeq); + this.updateSeq = updateSeq; + } + + private long getUpdateSeq() { + updateLock.readLock().lock(); + try { + return this.updateSeq; + } finally { + updateLock.readLock().unlock(); + } + } + +} \ No newline at end of file diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexFactory.java similarity index 50% copy from java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java copy to java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexFactory.java index bd31801fd..8d728c080 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexFactory.java @@ -14,27 +14,12 @@ package org.apache.couchdb.nouveau.core; import java.io.IOException; -import java.util.concurrent.Executor; +import java.nio.file.Path; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.SearcherFactory; +import org.apache.couchdb.nouveau.api.IndexDefinition; -public class ParallelSearcherFactory extends SearcherFactory { +public interface IndexFactory { - private Executor executor; - - public Executor getExecutor() { - return executor; - } - - public void setExecutor(Executor executor) { - this.executor = executor; - } - - @Override - public IndexSearcher newSearcher(final IndexReader reader, final IndexReader previousReader) throws IOException { - return new IndexSearcher(reader, executor); - } + Index open(final Path path, final IndexDefinition indexDefinition) throws IOException; } diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java index dd0f1f2e0..a5bb2970b 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java @@ -18,13 +18,7 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; -import java.util.Collections; -import java.util.Map; import java.util.concurrent.CompletionException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Stream; import javax.validation.constraints.Min; @@ -34,21 +28,13 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response.Status; import org.apache.couchdb.nouveau.api.IndexDefinition; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.misc.store.DirectIODirectory; -import org.apache.lucene.search.SearcherFactory; -import org.apache.lucene.search.SearcherManager; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.util.IOUtils; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.caffeine.MetricsStatsCounter; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.benmanes.caffeine.cache.CacheLoader; @@ -58,9 +44,6 @@ import com.github.benmanes.caffeine.cache.RemovalCause; import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Scheduler; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.caffeine.MetricsStatsCounter; - import io.dropwizard.lifecycle.Managed; public class IndexManager implements Managed { @@ -69,124 +52,6 @@ public class IndexManager implements Managed { private static final int RETRY_SLEEP_MS = 5; private static final Logger LOGGER = LoggerFactory.getLogger(IndexManager.class); - public class Index { - private static final String DEFAULT_FIELD = "default"; - private final String name; - private IndexWriter writer; - private SearcherManager searcherManager; - private Analyzer analyzer; - private final AtomicBoolean deleteOnClose = new AtomicBoolean(); - private final AtomicLong updateSeq = new AtomicLong(); - - // The write lock is to ensure there are no readers/searchers when - // we want to close the index. - private ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(); - private Lock rl = rwl.readLock(); - private Lock wl = rwl.writeLock(); - - private Index( - String name, - IndexWriter writer, - SearcherManager searcherManager, - Analyzer analyzer, - long updateSeq) { - this.name = name; - this.writer = writer; - this.searcherManager = searcherManager; - this.analyzer = analyzer; - this.updateSeq.set(updateSeq); - } - - public String getName() { - return name; - } - - public IndexWriter getWriter() { - return writer; - } - - public SearcherManager getSearcherManager() { - return searcherManager; - } - - public QueryParser getQueryParser() { - return new NouveauQueryParser(DEFAULT_FIELD, analyzer); - } - - public boolean commit() throws IOException { - rl.lock(); - try { - writer.setLiveCommitData(generateCommitData().entrySet()); - return writer.commit() != -1; - } finally { - rl.unlock(); - } - } - - public long getUpdateSeq() throws IOException { - return updateSeq.get(); - } - - public void incrementUpdateSeq(final long updateSeq) throws IOException { - final long newSeq = this.updateSeq.accumulateAndGet(updateSeq, (a, b) -> Math.max(a, b)); - if (newSeq != updateSeq) { - throw new UpdatesOutOfOrderException(); - } - } - - public void close() throws IOException { - wl.lock(); - try { - if (writer == null) { - // Already closed. - return; - } - - // Close searcher manager - if (searcherManager != null) { - try { - searcherManager.close(); - } catch (IOException e) { - LOGGER.info(this + " threw exception when closing searcherManager.", e); - } finally { - searcherManager = null; - } - } - - if (deleteOnClose.get()) { - try { - // No need to commit in this case. - writer.rollback(); - } catch (IOException e) { - LOGGER.info(this + " threw exception when rolling back writer.", e); - } finally { - writer = null; - } - IOUtils.rm(indexRootPath(name)); - } else { - try { - writer.setLiveCommitData(generateCommitData().entrySet()); - writer.close(); - LOGGER.info("{} closed.", this); - } finally { - writer = null; - } - } - } finally { - wl.unlock(); - } - } - - private Map<String, String> generateCommitData() { - return Collections.singletonMap("update_seq", Long.toString(updateSeq.get())); - } - - @Override - public String toString() { - return "Index [name=" + name + "]"; - } - } - private class IndexLoader implements CacheLoader<String, Index> { @Override @@ -237,12 +102,12 @@ public class IndexManager implements Managed { private Path rootDir; @NotNull - private AnalyzerFactory analyzerFactory; + private Lucene9AnalyzerFactory analyzerFactory; @NotNull private ObjectMapper objectMapper; - private SearcherFactory searcherFactory; + private IndexFactory indexFactory; private MetricRegistry metricRegistry; @@ -253,11 +118,11 @@ public class IndexManager implements Managed { final Index result = getFromCache(name); // Check if we're in the middle of closing. - result.rl.lock(); - if (result.writer != null) { + result.lock(); + if (!result.isClosed()) { return result; } - result.rl.unlock(); + result.unlock(); // Retry after a short sleep. try { @@ -271,7 +136,7 @@ public class IndexManager implements Managed { } public void release(final Index index) throws IOException { - index.rl.unlock(); + index.unlock(); } public void create(final String name, IndexDefinition indexDefinition) throws IOException { @@ -301,7 +166,7 @@ public class IndexManager implements Managed { private void deleteIndex(final String name) throws IOException { final Index index = acquire(name); try { - index.deleteOnClose.set(true); + index.setDeleteOnClose(true); cache.invalidate(name); } finally { release(index); @@ -344,7 +209,7 @@ public class IndexManager implements Managed { this.rootDir = rootDir; } - public void setAnalyzerFactory(final AnalyzerFactory analyzerFactory) { + public void setAnalyzerFactory(final Lucene9AnalyzerFactory analyzerFactory) { this.analyzerFactory = analyzerFactory; } @@ -352,8 +217,8 @@ public class IndexManager implements Managed { this.objectMapper = objectMapper; } - public void setSearcherFactory(final SearcherFactory searcherFactory) { - this.searcherFactory = searcherFactory; + public void setIndexFactory(final IndexFactory indexFactory) { + this.indexFactory = indexFactory; } public void setMetricRegistry(final MetricRegistry metricRegistry) { @@ -405,48 +270,9 @@ public class IndexManager implements Managed { } private Index openExistingIndex(final String name) throws IOException { - final IndexDefinition indexDefinition = objectMapper.readValue(indexDefinitionPath(name).toFile(), IndexDefinition.class); - final Analyzer analyzer = analyzerFactory.fromDefinition(indexDefinition); final Path path = indexPath(name); - final Directory dir = directory(path); - final IndexWriter writer = newWriter(dir, analyzer); - final SearcherManager searcherManager = new SearcherManager(writer, searcherFactory); - final long updateSeq = getUpdateSeq(writer); - return new Index(name, writer, searcherManager, analyzer, updateSeq); - } - - private long getUpdateSeq(final IndexWriter writer) throws IOException { - final Iterable<Map.Entry<String, String>> commitData = writer.getLiveCommitData(); - if (commitData == null) { - return 0L; - } - for (Map.Entry<String, String> entry : commitData) { - if (entry.getKey().equals("update_seq")) { - return Long.parseLong(entry.getValue()); - } - } - return 0L; - } - - private IndexWriter newWriter(final Directory dir, final Analyzer analyzer) throws IOException { - LockObtainFailedException exceptionThrown = null; - for (int i = 0; i < RETRY_LIMIT; i++) { - try { - final IndexWriterConfig config = new IndexWriterConfig(analyzer); - config.setCommitOnClose(true); - config.setUseCompoundFile(false); - return new IndexWriter(dir, config); - } catch (LockObtainFailedException e) { - exceptionThrown = e; - try { - Thread.sleep(RETRY_SLEEP_MS); - } catch (InterruptedException e1) { - Thread.interrupted(); - break; - } - } - } - throw exceptionThrown; + final IndexDefinition indexDefinition = objectMapper.readValue(indexDefinitionPath(name).toFile(), IndexDefinition.class); + return indexFactory.open(path, indexDefinition); } private boolean isIndex(final Path path) { @@ -470,8 +296,4 @@ public class IndexManager implements Managed { Status.BAD_REQUEST); } - private Directory directory(final Path path) throws IOException { - return new DirectIODirectory(FSDirectory.open(path)); - } - } diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/AnalyzerFactory.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactory.java similarity index 98% rename from java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/AnalyzerFactory.java rename to java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactory.java index 0ad6c0311..5544af267 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/AnalyzerFactory.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactory.java @@ -62,7 +62,10 @@ import org.apache.lucene.analysis.sv.SwedishAnalyzer; import org.apache.lucene.analysis.th.ThaiAnalyzer; import org.apache.lucene.analysis.tr.TurkishAnalyzer; -public class AnalyzerFactory { +public final class Lucene9AnalyzerFactory { + + public Lucene9AnalyzerFactory() { + } public Analyzer fromDefinition(final IndexDefinition indexDefinition) { final Analyzer defaultAnalyzer = newAnalyzer(indexDefinition.getDefaultAnalyzer()); diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/SearchResource.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9Index.java similarity index 57% copy from java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/SearchResource.java copy to java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9Index.java index dd78e861c..d62beff9b 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/SearchResource.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9Index.java @@ -11,10 +11,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.resources; +package org.apache.couchdb.nouveau.core; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -22,26 +24,17 @@ import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; -import javax.validation.Valid; -import javax.validation.constraints.NotNull; -import javax.ws.rs.Consumes; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response.Status; +import org.apache.couchdb.nouveau.api.DocumentDeleteRequest; +import org.apache.couchdb.nouveau.api.DocumentUpdateRequest; import org.apache.couchdb.nouveau.api.SearchHit; import org.apache.couchdb.nouveau.api.SearchRequest; import org.apache.couchdb.nouveau.api.SearchResults; -import org.apache.couchdb.nouveau.core.IndexManager; -import org.apache.couchdb.nouveau.core.IndexManager.Index; -import org.apache.couchdb.nouveau.core.QueryParserException; -import com.codahale.metrics.annotation.Timed; - +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; import org.apache.lucene.facet.FacetResult; import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.FacetsCollector; @@ -51,66 +44,100 @@ import org.apache.lucene.facet.StringDocValuesReaderState; import org.apache.lucene.facet.StringValueFacetCounts; import org.apache.lucene.facet.range.DoubleRange; import org.apache.lucene.facet.range.DoubleRangeFacetCounts; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.misc.store.DirectIODirectory; import org.apache.lucene.search.CollectorManager; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MultiCollectorManager; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.SearcherFactory; import org.apache.lucene.search.SearcherManager; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldCollector; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; -@Path("/index/{name}") -@Consumes(MediaType.APPLICATION_JSON) -@Produces(MediaType.APPLICATION_JSON) -public class SearchResource { +class Lucene9Index extends Index { private static final DoubleRange[] EMPTY_DOUBLE_RANGE_ARRAY = new DoubleRange[0]; private static final Sort DEFAULT_SORT = new Sort(SortField.FIELD_SCORE, new SortField("_id", SortField.Type.STRING)); private static final Pattern SORT_FIELD_RE = Pattern.compile("^([-+])?([\\.\\w]+)(?:<(\\w+)>)?$"); - private final IndexManager indexManager; - public SearchResource(final IndexManager indexManager) { - this.indexManager = indexManager; + private final Analyzer analyzer; + private final IndexWriter writer; + private final SearcherManager searcherManager; + + static Index open(final Path path, final Analyzer analyzer, final SearcherFactory searcherFactory) + throws IOException { + final Directory dir = new DirectIODirectory(FSDirectory.open(path)); + final IndexWriterConfig config = new IndexWriterConfig(analyzer); + config.setCommitOnClose(true); + config.setUseCompoundFile(false); + final IndexWriter writer = new IndexWriter(dir, config); + final long updateSeq = getUpdateSeq(writer); + final SearcherManager searcherManager = new SearcherManager(writer, searcherFactory); + return new Lucene9Index(analyzer, writer, updateSeq, searcherManager); } - @POST - @Timed - @Path("/search") - public SearchResults searchIndex(@PathParam("name") String name, @NotNull @Valid SearchRequest searchRequest) - throws IOException, QueryParserException { - final Index index = indexManager.acquire(name); - try { - final Query query = index.getQueryParser().parse(searchRequest); + private Lucene9Index(final Analyzer analyzer, final IndexWriter writer, final long updateSeq, + final SearcherManager searcherManager) { + super(updateSeq); + this.analyzer = analyzer; + this.writer = writer; + this.searcherManager = searcherManager; + } - // Construct CollectorManagers. - final MultiCollectorManager cm; - final CollectorManager<?, ? extends TopDocs> hits = hitCollector(searchRequest); + @Override + public int doNumDocs() throws IOException { + return writer.getDocStats().numDocs; + } - final SearcherManager searcherManager = index.getSearcherManager(); - searcherManager.maybeRefreshBlocking(); + @Override + public void doUpdate(final String docId, final DocumentUpdateRequest request) throws IOException { + final Term docIdTerm = docIdTerm(docId); + final Document doc = toDocument(docId, request); + writer.updateDocument(docIdTerm, doc); + } - final IndexSearcher searcher = searcherManager.acquire(); - try { - if (searchRequest.hasCounts() || searchRequest.hasRanges()) { - cm = new MultiCollectorManager(hits, new FacetsCollectorManager()); - } else { - cm = new MultiCollectorManager(hits); - } - final Object[] reduces = searcher.search(query, cm); - return toSearchResults(searchRequest, searcher, reduces); - } catch (IllegalStateException e) { - throw new WebApplicationException(e.getMessage(), e, Status.BAD_REQUEST); - } finally { - searcherManager.release(searcher); + @Override + public void doDelete(final String docId, final DocumentDeleteRequest request) throws IOException { + final Query query = docIdQuery(docId); + writer.deleteDocuments(query); + } + + @Override + public SearchResults doSearch(final SearchRequest request) throws IOException, QueryParserException { + final Query query = newQueryParser().parse(request); + + // Construct CollectorManagers. + final MultiCollectorManager cm; + final CollectorManager<?, ? extends TopDocs> hits = hitCollector(request); + + searcherManager.maybeRefreshBlocking(); + + final IndexSearcher searcher = searcherManager.acquire(); + try { + if (request.hasCounts() || request.hasRanges()) { + cm = new MultiCollectorManager(hits, new FacetsCollectorManager()); + } else { + cm = new MultiCollectorManager(hits); } + final Object[] reduces = searcher.search(query, cm); + return toSearchResults(request, searcher, reduces); + } catch (IllegalStateException e) { + throw new WebApplicationException(e.getMessage(), e, Status.BAD_REQUEST); } finally { - indexManager.release(index); + searcherManager.release(searcher); } } @@ -214,12 +241,12 @@ public class SearchResource { final List<String> sort = new ArrayList<String>(searchRequest.getSort()); final String last = sort.get(sort.size() - 1); // Append _id field if not already present. - switch(last) { + switch (last) { case "-_id<string>": case "_id<string>": break; default: - sort.add("_id<string>"); + sort.add("_id<string>"); } return convertSort(sort); } @@ -246,4 +273,71 @@ public class SearchResource { return new SortField(m.group(2), type, reverse); } + @Override + public boolean doCommit(final long updateSeq) throws IOException { + writer.setLiveCommitData(Collections.singletonMap("update_seq", Long.toString(updateSeq)).entrySet()); + return writer.commit() != -1; + } + + @Override + public void doClose(final boolean deleteOnClose) throws IOException { + if (deleteOnClose) { + // No need to commit in this case. + writer.rollback(); + final Directory dir = writer.getDirectory(); + for (final String name : dir.listAll()) { + dir.deleteFile(name); + } + } + writer.close(); + } + + private static Document toDocument(final String docId, final DocumentUpdateRequest request) throws IOException { + final Document result = new Document(); + + // id + result.add(new org.apache.lucene.document.StringField("_id", docId, Store.YES)); + result.add(new org.apache.lucene.document.SortedDocValuesField("_id", new BytesRef(docId))); + + // partition (optional) + if (request.hasPartition()) { + result.add(new org.apache.lucene.document.StringField("_partition", request.getPartition(), Store.NO)); + } + + for (IndexableField field : request.getFields()) { + // Underscore-prefix is reserved. + if (field.name().startsWith("_")) { + continue; + } + result.add(field); + } + + return result; + } + + private static Query docIdQuery(final String docId) { + return new TermQuery(docIdTerm(docId)); + } + + private static Term docIdTerm(final String docId) { + return new Term("_id", docId); + } + + private static long getUpdateSeq(final IndexWriter writer) throws IOException { + final Iterable<Map.Entry<String, String>> commitData = writer.getLiveCommitData(); + if (commitData == null) { + return 0L; + } + for (Map.Entry<String, String> entry : commitData) { + if (entry.getKey().equals("update_seq")) { + return Long.parseLong(entry.getValue()); + } + } + return 0L; + } + + public QueryParser newQueryParser() { + return new NouveauQueryParser("default", analyzer); + } + } diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9IndexFactory.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9IndexFactory.java new file mode 100644 index 000000000..016f427c1 --- /dev/null +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9IndexFactory.java @@ -0,0 +1,43 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.core; + +import java.io.IOException; +import java.nio.file.Path; + +import org.apache.couchdb.nouveau.api.IndexDefinition; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.search.SearcherFactory; + +public class Lucene9IndexFactory implements IndexFactory { + + private Lucene9AnalyzerFactory analyzerFactory; + + private SearcherFactory searcherFactory; + + public void setAnalyzerFactory(final Lucene9AnalyzerFactory analyzerFactory) { + this.analyzerFactory = analyzerFactory; + } + + public void setSearcherFactory(final SearcherFactory searcherFactory) { + this.searcherFactory = searcherFactory; + } + + @Override + public Index open(Path path, final IndexDefinition indexDefinition) throws IOException { + final Analyzer analyzer = analyzerFactory.fromDefinition(indexDefinition); + return Lucene9Index.open(path, analyzer, searcherFactory); + } + +} diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9ParallelSearcherFactory.java similarity index 94% rename from java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java rename to java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9ParallelSearcherFactory.java index bd31801fd..7021f2997 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/ParallelSearcherFactory.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/core/Lucene9ParallelSearcherFactory.java @@ -20,7 +20,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SearcherFactory; -public class ParallelSearcherFactory extends SearcherFactory { +public class Lucene9ParallelSearcherFactory extends SearcherFactory { private Executor executor; diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/health/IndexManagerHealthCheck.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/health/IndexManagerHealthCheck.java index b2653db38..e76d8da9c 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/health/IndexManagerHealthCheck.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/health/IndexManagerHealthCheck.java @@ -13,16 +13,17 @@ package org.apache.couchdb.nouveau.health; +import static org.apache.couchdb.nouveau.api.LuceneVersion.LUCENE_9; + import java.io.IOException; +import java.util.Collections; +import org.apache.couchdb.nouveau.api.DocumentUpdateRequest; import org.apache.couchdb.nouveau.api.IndexDefinition; -import static org.apache.couchdb.nouveau.api.LuceneVersion.*; +import org.apache.couchdb.nouveau.core.Index; import org.apache.couchdb.nouveau.core.IndexManager; -import org.apache.couchdb.nouveau.core.IndexManager.Index; -import com.codahale.metrics.health.HealthCheck; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.IndexWriter; +import com.codahale.metrics.health.HealthCheck; public class IndexManagerHealthCheck extends HealthCheck { @@ -44,14 +45,14 @@ public class IndexManagerHealthCheck extends HealthCheck { indexManager.create(name, new IndexDefinition(LUCENE_9, "standard", null)); final Index index = indexManager.acquire(name); try { - final IndexWriter writer = index.getWriter(); - try { - writer.addDocument(new Document()); - writer.commit(); - return Result.healthy(); - } finally { - indexManager.deleteAll(name); - } + final DocumentUpdateRequest request = new DocumentUpdateRequest(1, null, Collections.emptyList()); + index.update("foo", request); + index.commit(); + index.setDeleteOnClose(true); + index.close(); + return Result.healthy(); + } catch (final IOException e) { + return Result.unhealthy(e); } finally { indexManager.release(index); } diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java index 60e8c8ca3..c495cf5fa 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java @@ -29,7 +29,7 @@ import javax.ws.rs.core.Response.Status; import org.apache.couchdb.nouveau.api.AnalyzeRequest; import org.apache.couchdb.nouveau.api.AnalyzeResponse; -import org.apache.couchdb.nouveau.core.AnalyzerFactory; +import org.apache.couchdb.nouveau.core.Lucene9AnalyzerFactory; import com.codahale.metrics.annotation.Timed; import org.apache.lucene.analysis.Analyzer; @@ -41,9 +41,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @Produces(MediaType.APPLICATION_JSON) public class AnalyzeResource { - private final AnalyzerFactory analyzerFactory; + private final Lucene9AnalyzerFactory analyzerFactory; - public AnalyzeResource(AnalyzerFactory analyzerFactory) { + public AnalyzeResource(Lucene9AnalyzerFactory analyzerFactory) { this.analyzerFactory = analyzerFactory; } diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java index cd10226db..ca7ee15c0 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java @@ -30,15 +30,10 @@ import org.apache.couchdb.nouveau.api.DocumentDeleteRequest; import org.apache.couchdb.nouveau.api.DocumentUpdateRequest; import org.apache.couchdb.nouveau.api.IndexDefinition; import org.apache.couchdb.nouveau.api.IndexInfo; -import org.apache.couchdb.nouveau.core.DocumentFactory; +import org.apache.couchdb.nouveau.core.Index; import org.apache.couchdb.nouveau.core.IndexManager; -import org.apache.couchdb.nouveau.core.IndexManager.Index; -import com.codahale.metrics.annotation.Timed; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.TermQuery; +import com.codahale.metrics.annotation.Timed; @Path("/index/{name}") @Consumes(MediaType.APPLICATION_JSON) @@ -46,26 +41,20 @@ import org.apache.lucene.search.TermQuery; public class IndexResource { private final IndexManager indexManager; - private final DocumentFactory documentFactory; - public IndexResource(final IndexManager indexManager, final DocumentFactory documentFactory) { + public IndexResource(final IndexManager indexManager) { this.indexManager = indexManager; - this.documentFactory = documentFactory; } @GET @SuppressWarnings("resource") public IndexInfo indexInfo(@PathParam("name") String name) throws IOException { - final long updateSeq; - final int numDocs; final Index index = indexManager.acquire(name); try { - updateSeq = index.getUpdateSeq(); - numDocs = index.getWriter().getDocStats().numDocs; + return index.info(); } finally { indexManager.release(index); } - return new IndexInfo(updateSeq, numDocs); } @DELETE @@ -84,9 +73,7 @@ public class IndexResource { public void deleteDoc(@PathParam("name") String name, @PathParam("docId") String docId, @NotNull @Valid final DocumentDeleteRequest request) throws IOException { final Index index = indexManager.acquire(name); try { - final IndexWriter writer = index.getWriter(); - writer.deleteDocuments(new TermQuery(new Term("_id", docId))); - index.incrementUpdateSeq(request.getSeq()); + index.delete(docId, request); } finally { indexManager.release(index); } @@ -98,10 +85,7 @@ public class IndexResource { public void updateDoc(@PathParam("name") String name, @PathParam("docId") String docId, @NotNull @Valid final DocumentUpdateRequest request) throws IOException { final Index index = indexManager.acquire(name); try { - final IndexWriter writer = index.getWriter(); - final Document doc = documentFactory.build(docId, request); - writer.updateDocument(new Term("_id", docId), doc); - index.incrementUpdateSeq(request.getSeq()); + index.update(docId, request); } finally { indexManager.release(index); } diff --git a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/SearchResource.java b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/SearchResource.java index dd78e861c..03f6c0c20 100644 --- a/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/SearchResource.java +++ b/java/nouveau/server/src/main/java/org/apache/couchdb/nouveau/resources/SearchResource.java @@ -14,13 +14,6 @@ package org.apache.couchdb.nouveau.resources; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import javax.validation.Valid; import javax.validation.constraints.NotNull; @@ -29,50 +22,21 @@ import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; -import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response.Status; -import org.apache.couchdb.nouveau.api.SearchHit; import org.apache.couchdb.nouveau.api.SearchRequest; import org.apache.couchdb.nouveau.api.SearchResults; +import org.apache.couchdb.nouveau.core.Index; import org.apache.couchdb.nouveau.core.IndexManager; -import org.apache.couchdb.nouveau.core.IndexManager.Index; import org.apache.couchdb.nouveau.core.QueryParserException; -import com.codahale.metrics.annotation.Timed; -import org.apache.lucene.document.Document; -import org.apache.lucene.facet.FacetResult; -import org.apache.lucene.facet.Facets; -import org.apache.lucene.facet.FacetsCollector; -import org.apache.lucene.facet.FacetsCollectorManager; -import org.apache.lucene.facet.LabelAndValue; -import org.apache.lucene.facet.StringDocValuesReaderState; -import org.apache.lucene.facet.StringValueFacetCounts; -import org.apache.lucene.facet.range.DoubleRange; -import org.apache.lucene.facet.range.DoubleRangeFacetCounts; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.search.CollectorManager; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MultiCollectorManager; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.SearcherManager; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopFieldCollector; +import com.codahale.metrics.annotation.Timed; @Path("/index/{name}") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public class SearchResource { - private static final DoubleRange[] EMPTY_DOUBLE_RANGE_ARRAY = new DoubleRange[0]; - private static final Sort DEFAULT_SORT = new Sort(SortField.FIELD_SCORE, - new SortField("_id", SortField.Type.STRING)); - private static final Pattern SORT_FIELD_RE = Pattern.compile("^([-+])?([\\.\\w]+)(?:<(\\w+)>)?$"); private final IndexManager indexManager; public SearchResource(final IndexManager indexManager) { @@ -86,164 +50,10 @@ public class SearchResource { throws IOException, QueryParserException { final Index index = indexManager.acquire(name); try { - final Query query = index.getQueryParser().parse(searchRequest); - - // Construct CollectorManagers. - final MultiCollectorManager cm; - final CollectorManager<?, ? extends TopDocs> hits = hitCollector(searchRequest); - - final SearcherManager searcherManager = index.getSearcherManager(); - searcherManager.maybeRefreshBlocking(); - - final IndexSearcher searcher = searcherManager.acquire(); - try { - if (searchRequest.hasCounts() || searchRequest.hasRanges()) { - cm = new MultiCollectorManager(hits, new FacetsCollectorManager()); - } else { - cm = new MultiCollectorManager(hits); - } - final Object[] reduces = searcher.search(query, cm); - return toSearchResults(searchRequest, searcher, reduces); - } catch (IllegalStateException e) { - throw new WebApplicationException(e.getMessage(), e, Status.BAD_REQUEST); - } finally { - searcherManager.release(searcher); - } + return index.search(searchRequest); } finally { indexManager.release(index); } } - private CollectorManager<?, ? extends TopDocs> hitCollector(final SearchRequest searchRequest) { - final Sort sort = toSort(searchRequest); - - final FieldDoc after = searchRequest.getAfter(); - if (after != null) { - if (getLastSortField(sort).getReverse()) { - after.doc = 0; - } else { - after.doc = Integer.MAX_VALUE; - } - } - - return TopFieldCollector.createSharedManager( - sort, - searchRequest.getLimit(), - after, - 1000); - } - - private SortField getLastSortField(final Sort sort) { - final SortField[] sortFields = sort.getSort(); - return sortFields[sortFields.length - 1]; - } - - private SearchResults toSearchResults(final SearchRequest searchRequest, final IndexSearcher searcher, - final Object[] reduces) throws IOException { - final SearchResults result = new SearchResults(); - collectHits(searcher, (TopDocs) reduces[0], result); - if (reduces.length == 2) { - collectFacets(searchRequest, searcher, (FacetsCollector) reduces[1], result); - } - return result; - } - - private void collectHits(final IndexSearcher searcher, final TopDocs topDocs, final SearchResults searchResults) - throws IOException { - final List<SearchHit> hits = new ArrayList<SearchHit>(topDocs.scoreDocs.length); - - for (final ScoreDoc scoreDoc : topDocs.scoreDocs) { - final Document doc = searcher.doc(scoreDoc.doc); - - final List<IndexableField> fields = new ArrayList<IndexableField>(doc.getFields()); - for (IndexableField field : doc.getFields()) { - if (field.name().equals("_id")) { - fields.remove(field); - } - } - - hits.add(new SearchHit(doc.get("_id"), (FieldDoc) scoreDoc, fields)); - } - - searchResults.setTotalHits(topDocs.totalHits); - searchResults.setHits(hits); - } - - private void collectFacets(final SearchRequest searchRequest, final IndexSearcher searcher, - final FacetsCollector fc, final SearchResults searchResults) throws IOException { - if (searchRequest.hasCounts()) { - final Map<String, Map<String, Number>> countsMap = new HashMap<String, Map<String, Number>>( - searchRequest.getCounts().size()); - for (final String field : searchRequest.getCounts()) { - final StringDocValuesReaderState state = new StringDocValuesReaderState(searcher.getIndexReader(), - field); - final StringValueFacetCounts counts = new StringValueFacetCounts(state, fc); - countsMap.put(field, collectFacets(counts, searchRequest.getTopN(), field)); - } - searchResults.setCounts(countsMap); - } - - if (searchRequest.hasRanges()) { - final Map<String, Map<String, Number>> rangesMap = new HashMap<String, Map<String, Number>>( - searchRequest.getRanges().size()); - for (final Entry<String, List<DoubleRange>> entry : searchRequest.getRanges().entrySet()) { - final DoubleRangeFacetCounts counts = new DoubleRangeFacetCounts(entry.getKey(), fc, - entry.getValue().toArray(EMPTY_DOUBLE_RANGE_ARRAY)); - rangesMap.put(entry.getKey(), collectFacets(counts, searchRequest.getTopN(), entry.getKey())); - } - searchResults.setRanges(rangesMap); - } - } - - private Map<String, Number> collectFacets(final Facets facets, final int topN, final String dim) - throws IOException { - final FacetResult topChildren = facets.getTopChildren(topN, dim); - final Map<String, Number> result = new HashMap<String, Number>(topChildren.childCount); - for (final LabelAndValue lv : topChildren.labelValues) { - result.put(lv.label, lv.value); - } - return result; - } - - // Ensure _id is final sort field so we can paginate. - private Sort toSort(final SearchRequest searchRequest) { - if (!searchRequest.hasSort()) { - return DEFAULT_SORT; - } - - final List<String> sort = new ArrayList<String>(searchRequest.getSort()); - final String last = sort.get(sort.size() - 1); - // Append _id field if not already present. - switch(last) { - case "-_id<string>": - case "_id<string>": - break; - default: - sort.add("_id<string>"); - } - return convertSort(sort); - } - - private Sort convertSort(final List<String> sort) { - final SortField[] fields = new SortField[sort.size()]; - for (int i = 0; i < sort.size(); i++) { - fields[i] = convertSortField(sort.get(i)); - } - return new Sort(fields); - } - - private SortField convertSortField(final String sortString) { - final Matcher m = SORT_FIELD_RE.matcher(sortString); - if (!m.matches()) { - throw new WebApplicationException( - sortString + " is not a valid sort parameter", Status.BAD_REQUEST); - } - final boolean reverse = "-".equals(m.group(1)); - SortField.Type type = SortField.Type.DOUBLE; - if ("string".equals(m.group(3))) { - type = SortField.Type.STRING; - } - return new SortField(m.group(2), type, reverse); - } - } diff --git a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/IntegrationTest.java b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/IntegrationTest.java index 8b2d7029a..66f774514 100644 --- a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/IntegrationTest.java +++ b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/IntegrationTest.java @@ -13,8 +13,10 @@ package org.apache.couchdb.nouveau; +import static org.apache.couchdb.nouveau.api.LuceneVersion.LUCENE_9; import static org.assertj.core.api.Assertions.assertThat; +import java.io.IOException; import java.util.List; import java.util.Map; @@ -24,10 +26,8 @@ import javax.ws.rs.core.Response; import org.apache.couchdb.nouveau.api.DocumentUpdateRequest; import org.apache.couchdb.nouveau.api.IndexDefinition; -import static org.apache.couchdb.nouveau.api.LuceneVersion.*; import org.apache.couchdb.nouveau.api.SearchRequest; import org.apache.couchdb.nouveau.api.SearchResults; - import org.apache.lucene.document.DoubleDocValuesField; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.SortedSetDocValuesField; @@ -112,12 +112,16 @@ public class IntegrationTest { } @Test - public void healthCheckShouldSucceed() { + public void healthCheckShouldSucceed() throws IOException { final Response healthCheckResponse = APP.client().target("http://localhost:"; + APP.getAdminPort() + "/healthcheck") .request() .get(); + //healthCheckResponse.bufferEntity(); + //InputStream in = (InputStream) healthCheckResponse.getEntity(); + //System.err.printf("health check response: %s\n", new String(in.readAllBytes())); + assertThat(healthCheckResponse) .extracting(Response::getStatus) .isEqualTo(Response.Status.OK.getStatusCode()); diff --git a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java index 9544b6f6e..7bc886af2 100644 --- a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java +++ b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java @@ -7,12 +7,12 @@ import java.util.List; import java.util.Map; import org.apache.couchdb.nouveau.core.ser.LuceneModule; -import com.fasterxml.jackson.databind.ObjectMapper; - import org.apache.lucene.facet.range.DoubleRange; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import com.fasterxml.jackson.databind.ObjectMapper; + public class SearchRequestTest { private static ObjectMapper mapper; diff --git a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java index 64081d5c0..4487b69e3 100644 --- a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java +++ b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java @@ -42,7 +42,7 @@ public class IndexManagerTest { public void setup() throws Exception { manager = new IndexManager(); manager.setMetricRegistry(new MetricRegistry()); - manager.setAnalyzerFactory(new AnalyzerFactory()); + manager.setAnalyzerFactory(new Lucene9AnalyzerFactory()); manager.setCommitIntervalSeconds(5); manager.setObjectMapper(new ObjectMapper()); manager.setRootDir(tempDir); diff --git a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/AnalyzerFactoryTest.java b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactoryTest.java similarity index 98% rename from java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/AnalyzerFactoryTest.java rename to java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactoryTest.java index 43bdb4e14..0adf18d45 100644 --- a/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/AnalyzerFactoryTest.java +++ b/java/nouveau/server/src/test/java/org/apache/couchdb/nouveau/core/Lucene9AnalyzerFactoryTest.java @@ -56,7 +56,7 @@ import org.apache.lucene.analysis.th.ThaiAnalyzer; import org.apache.lucene.analysis.tr.TurkishAnalyzer; import org.junit.jupiter.api.Test; -public class AnalyzerFactoryTest { +public class Lucene9AnalyzerFactoryTest { @Test public void testkeyword() throws Exception { @@ -249,7 +249,7 @@ public class AnalyzerFactoryTest { } private void assertAnalyzer(final String name, final Class<? extends Analyzer> clazz) throws Exception { - final AnalyzerFactory factory = new AnalyzerFactory(); + final Lucene9AnalyzerFactory factory = new Lucene9AnalyzerFactory(); assertThat(factory.newAnalyzer(name)).isInstanceOf(clazz); }
