Author: tommaso
Date: Mon May 21 17:06:10 2018
New Revision: 1831979
URL: http://svn.apache.org/viewvc?rev=1831979&view=rev
Log:
OAK-7410 - first draft of abstract editor
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ExtractedTextCache.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ReindexOperations.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/TextExtractionStatsMBean.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/BlobByteSource.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/DocumentMaker.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextBinaryTextExtractor.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriter.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriterFactory.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TextExtractionStats.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TikaParserConfig.java
(with props)
jackrabbit/oak/trunk/oak-search/src/main/resources/
jackrabbit/oak/trunk/oak-search/src/main/resources/org/
jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/
jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/jackrabbit/
jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/jackrabbit/oak/
jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/jackrabbit/oak/plugins/
jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/jackrabbit/oak/plugins/index/
jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/jackrabbit/oak/plugins/index/search/
jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/jackrabbit/oak/plugins/index/search/tika-config.xml
(with props)
Modified:
jackrabbit/oak/trunk/oak-search/pom.xml
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/Aggregate.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/BadIndexTracker.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/NodeStateCloner.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/PropertyDefinition.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/IndexHelper.java
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/TestUtil.java
Modified: jackrabbit/oak/trunk/oak-search/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/pom.xml?rev=1831979&r1=1831978&r2=1831979&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-search/pom.xml Mon May 21 17:06:10 2018
@@ -112,6 +112,11 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>${tika.version}</version>
+ </dependency>
<!-- Logging -->
<dependency>
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/Aggregate.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/Aggregate.java?rev=1831979&r1=1831978&r2=1831979&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/Aggregate.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/Aggregate.java
Mon May 21 17:06:10 2018
@@ -47,7 +47,7 @@ import static com.google.common.collect.
import static org.apache.jackrabbit.oak.commons.PathUtils.elements;
import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath;
-class Aggregate {
+public class Aggregate {
public static final String MATCH_ALL = "*";
@@ -425,9 +425,9 @@ class Aggregate {
}
public static class NodeIncludeResult {
- final NodeState nodeState;
- final String nodePath;
- final String rootIncludePath;
+ public final NodeState nodeState;
+ public final String nodePath;
+ public final String rootIncludePath;
public NodeIncludeResult(String nodePath, NodeState nodeState) {
this(nodePath, null, nodeState);
@@ -453,9 +453,9 @@ class Aggregate {
}
public static class PropertyIncludeResult {
- final PropertyState propertyState;
- final PropertyDefinition pd;
- final String propertyPath;
+ public final PropertyState propertyState;
+ public final PropertyDefinition pd;
+ public final String propertyPath;
final String nodePath;
public PropertyIncludeResult(PropertyState propertyState,
PropertyDefinition pd,
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/BadIndexTracker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/BadIndexTracker.java?rev=1831979&r1=1831978&r2=1831979&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/BadIndexTracker.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/BadIndexTracker.java
Mon May 21 17:06:10 2018
@@ -81,10 +81,10 @@ class BadIndexTracker {
BadIndexInfo badIndex = badPersistedIndexes.get(path);
if (badIndex == null) {
badPersistedIndexes.put(path, new BadIndexInfo(path, e, true));
- log.error("Could not open the Lucene index at [{}]", path, e);
+ log.error("Could not open the Fulltext index at [{}]", path, e);
} else {
badIndex.failedAccess(e);
- log.error("Could not open the Lucene index at [{}] . {}",
+ log.error("Could not open the Fulltext index at [{}] . {}",
path, badIndex.getStats(), e);
}
}
@@ -97,10 +97,10 @@ class BadIndexTracker {
BadIndexInfo badIndex = badIndexesForRead.get(path);
if (badIndex == null) {
badIndexesForRead.put(path, new BadIndexInfo(path, e, false));
- log.error("Could not access the Lucene index at [{}]", path, e);
+ log.error("Could not access the Fulltext index at [{}]", path, e);
} else {
badIndex.failedAccess(e);
- log.error("Could not access the Lucene index at [{}] . {}",
+ log.error("Could not access the Fulltext index at [{}] . {}",
path, badIndex.getStats(), e);
}
}
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ExtractedTextCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ExtractedTextCache.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ExtractedTextCache.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ExtractedTextCache.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,408 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Weigher;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.cache.CacheStats;
+import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.jackrabbit.oak.plugins.index.fulltext.ExtractedText;
+import
org.apache.jackrabbit.oak.plugins.index.fulltext.ExtractedText.ExtractionResult;
+import
org.apache.jackrabbit.oak.plugins.index.fulltext.PreExtractedTextProvider;
+import
org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexEditor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
+
+public class ExtractedTextCache {
+ private static final boolean CACHE_ONLY_SUCCESS =
+ Boolean.getBoolean("oak.extracted.cacheOnlySuccess");
+ private static final int EXTRACTION_TIMEOUT_SECONDS =
+ Integer.getInteger("oak.extraction.timeoutSeconds", 60);
+ private static final int EXTRACTION_MAX_THREADS =
+ Integer.getInteger("oak.extraction.maxThreads", 10);
+ private static final boolean EXTRACT_IN_CALLER_THREAD =
+ Boolean.getBoolean("oak.extraction.inCallerThread");
+ private static final boolean EXTRACT_FORGET_TIMEOUT =
+ Boolean.getBoolean("oak.extraction.forgetTimeout");
+
+ private static final String TIMEOUT_MAP =
"textExtractionTimeout.properties";
+ private static final String EMPTY_STRING = "";
+ private static final Logger log =
LoggerFactory.getLogger(ExtractedTextCache.class);
+ private volatile PreExtractedTextProvider extractedTextProvider;
+ private int textExtractionCount;
+ private long totalBytesRead;
+ private long totalTextSize;
+ private long totalTime;
+ private int preFetchedCount;
+ private final Cache<String, String> cache;
+ private final ConcurrentHashMap<String, String> timeoutMap;
+ private final File indexDir;
+ private final CacheStats cacheStats;
+ private final boolean alwaysUsePreExtractedCache;
+ private volatile ExecutorService executorService;
+ private volatile int timeoutCount;
+ private long extractionTimeoutMillis = EXTRACTION_TIMEOUT_SECONDS * 1000;
+
+ public ExtractedTextCache(long maxWeight, long expiryTimeInSecs){
+ this(maxWeight, expiryTimeInSecs, false, null);
+ }
+
+ public ExtractedTextCache(long maxWeight, long expiryTimeInSecs, boolean
alwaysUsePreExtractedCache,
+ File indexDir) {
+ if (maxWeight > 0) {
+ cache = CacheBuilder.newBuilder()
+ .weigher(EmpiricalWeigher.INSTANCE)
+ .maximumWeight(maxWeight)
+ .expireAfterAccess(expiryTimeInSecs, TimeUnit.SECONDS)
+ .recordStats()
+ .build();
+ cacheStats = new CacheStats(cache, "ExtractedTextCache",
+ EmpiricalWeigher.INSTANCE, maxWeight);
+ } else {
+ cache = null;
+ cacheStats = null;
+ }
+ this.alwaysUsePreExtractedCache = alwaysUsePreExtractedCache;
+ this.timeoutMap = new ConcurrentHashMap<String, String>();
+ this.indexDir = indexDir;
+ loadTimeoutMap();
+ }
+
+ /**
+ * Get the pre extracted text for given blob
+ * @return null if no pre extracted text entry found. Otherwise returns
the pre extracted
+ * text
+ */
+ @CheckForNull
+ public String get(String nodePath, String propertyName, Blob blob, boolean
reindexMode){
+ String result = null;
+ //Consult the PreExtractedTextProvider only in reindex mode and not in
+ //incremental indexing mode. As that would only contain older entries
+ //That also avoid loading on various state (See DataStoreTextWriter)
+ String propertyPath = concat(nodePath, propertyName);
+ log.trace("Looking for extracted text for [{}] with blobId [{}]",
propertyPath, blob.getContentIdentity());
+ if ((reindexMode || alwaysUsePreExtractedCache) &&
extractedTextProvider != null){
+ try {
+ ExtractedText text =
extractedTextProvider.getText(propertyPath, blob);
+ if (text != null) {
+ preFetchedCount++;
+ result = getText(text);
+ }
+ } catch (IOException e) {
+ log.warn("Error occurred while fetching pre extracted text for
{}", propertyPath, e);
+ }
+ }
+ String id = blob.getContentIdentity();
+ if (cache != null && id != null && result == null) {
+ result = cache.getIfPresent(id);
+ }
+ if (result == null && id != null) {
+ result = timeoutMap.get(id);
+ }
+ return result;
+ }
+
+ public void put(@Nonnull Blob blob, @Nonnull ExtractedText extractedText) {
+ String id = blob.getContentIdentity();
+ if (cache != null && id != null) {
+ if (extractedText.getExtractionResult() == ExtractionResult.SUCCESS
+ || !CACHE_ONLY_SUCCESS) {
+ cache.put(id, getText(extractedText));
+ }
+ }
+ }
+
+ public void putTimeout(@Nonnull Blob blob, @Nonnull ExtractedText
extractedText) {
+ if (EXTRACT_FORGET_TIMEOUT) {
+ return;
+ }
+ String id = blob.getContentIdentity();
+ timeoutMap.put(id, getText(extractedText));
+ storeTimeoutMap();
+ }
+
+ private static String getText(ExtractedText text) {
+ switch (text.getExtractionResult()) {
+ case SUCCESS:
+ return text.getExtractedText().toString();
+ case ERROR:
+ return FulltextIndexEditor.TEXT_EXTRACTION_ERROR;
+ case EMPTY:
+ return EMPTY_STRING;
+ }
+ throw new IllegalArgumentException();
+ }
+
+ public void addStats(int count, long timeInMillis, long bytesRead, long
textLength){
+ this.textExtractionCount += count;
+ this.totalTime += timeInMillis;
+ this.totalBytesRead += bytesRead;
+ this.totalTextSize += textLength;
+ }
+
+ public TextExtractionStatsMBean getStatsMBean() {
+ return new TextExtractionStatsMBean() {
+ @Override
+ public boolean isPreExtractedTextProviderConfigured() {
+ return extractedTextProvider != null;
+ }
+
+ @Override
+ public int getTextExtractionCount() {
+ return textExtractionCount;
+ }
+
+ @Override
+ public long getTotalTime() {
+ return totalTime;
+ }
+
+ @Override
+ public int getPreFetchedCount() {
+ return preFetchedCount;
+ }
+
+ @Override
+ public String getExtractedTextSize() {
+ return IOUtils.humanReadableByteCount(totalTextSize);
+ }
+
+ @Override
+ public String getBytesRead() {
+ return IOUtils.humanReadableByteCount(totalBytesRead);
+ }
+
+ @Override
+ public boolean isAlwaysUsePreExtractedCache() {
+ return alwaysUsePreExtractedCache;
+ }
+
+ @Override
+ public int getTimeoutCount() {
+ return timeoutCount;
+ }
+ };
+ }
+
+ @CheckForNull
+ public CacheStats getCacheStats() {
+ return cacheStats;
+ }
+
+ public void setExtractedTextProvider(PreExtractedTextProvider
extractedTextProvider) {
+ this.extractedTextProvider = extractedTextProvider;
+ }
+
+ public PreExtractedTextProvider getExtractedTextProvider() {
+ return extractedTextProvider;
+ }
+
+ void resetCache(){
+ if (cache != null){
+ cache.invalidateAll();
+ }
+ }
+
+ boolean isAlwaysUsePreExtractedCache() {
+ return alwaysUsePreExtractedCache;
+ }
+
+ //Taken from DocumentNodeStore and cache packages as they are private
+ private static class EmpiricalWeigher implements Weigher<String, String> {
+ public static final EmpiricalWeigher INSTANCE = new EmpiricalWeigher();
+
+ private EmpiricalWeigher() {
+ }
+
+ private static long getMemory(@Nonnull String s) {
+ return 16 // shallow size
+ + 40 + (long)s.length() * 2; // value
+ }
+
+ @Override
+ public int weigh(String key, String value) {
+ long size = 168; // overhead for each cache entry
+ size += getMemory(key); // key
+ size += getMemory(value); // value
+ if (size > Integer.MAX_VALUE) {
+ log.debug("Calculated weight larger than Integer.MAX_VALUE:
{}.", size);
+ size = Integer.MAX_VALUE;
+ }
+ return (int) size;
+ }
+ }
+
+ public void close() {
+ resetCache();
+ // don't clean the persistent map on purpose, so we don't re-try
+ // after restarting the service or so
+ closeExecutorService();
+ }
+
+ public void process(String name, Callable<Void> callable) throws
InterruptedException, Throwable {
+ Callable<Void> callable2 = new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ Thread t = Thread.currentThread();
+ String oldThreadName = t.getName();
+ t.setName(oldThreadName + ": " + name);
+ try {
+ return callable.call();
+ } finally {
+ Thread.currentThread().setName(oldThreadName);
+ }
+ }
+ };
+ try {
+ if (EXTRACT_IN_CALLER_THREAD) {
+ callable2.call();
+ } else {
+ Future<Void> future = getExecutor().submit(callable2);
+ future.get(extractionTimeoutMillis, TimeUnit.MILLISECONDS);
+ }
+ } catch (TimeoutException e) {
+ timeoutCount++;
+ throw e;
+ } catch (InterruptedException e) {
+ throw e;
+ } catch (ExecutionException e) {
+ throw e.getCause();
+ }
+ }
+
+ public void setExtractionTimeoutMillis(int extractionTimeoutMillis) {
+ this.extractionTimeoutMillis = extractionTimeoutMillis;
+ }
+
+ private ExecutorService getExecutor() {
+ if (executorService == null) {
+ createExecutor();
+ }
+ return executorService;
+ }
+
+ private synchronized void createExecutor() {
+ if (executorService != null) {
+ return;
+ }
+ log.debug("ExtractedTextCache createExecutor " + this);
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(1,
EXTRACTION_MAX_THREADS,
+ 60L, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>(), new ThreadFactory() {
+ private final AtomicInteger counter = new AtomicInteger();
+ private final Thread.UncaughtExceptionHandler handler = new
Thread.UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ log.warn("Error occurred in asynchronous processing ", e);
+ }
+ };
+ @Override
+ public Thread newThread(@Nonnull Runnable r) {
+ Thread thread = new Thread(r, createName());
+ thread.setDaemon(true);
+ thread.setPriority(Thread.MIN_PRIORITY);
+ thread.setUncaughtExceptionHandler(handler);
+ return thread;
+ }
+
+ private String createName() {
+ int index = counter.getAndIncrement();
+ return "oak binary text extractor" + (index == 0 ? "" : " " +
index);
+ }
+ });
+ executor.setKeepAliveTime(1, TimeUnit.MINUTES);
+ executor.allowCoreThreadTimeOut(true);
+ executorService = executor;
+ }
+
+ private synchronized void closeExecutorService() {
+ if (executorService != null) {
+ log.debug("ExtractedTextCache closeExecutorService " + this);
+ executorService.shutdown();
+ try {
+ executorService.awaitTermination(1, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ log.warn("Interrupted", e);
+ }
+ executorService = null;
+ }
+ }
+
+ private synchronized void loadTimeoutMap() {
+ if (indexDir == null || !indexDir.exists()) {
+ return;
+ }
+ File file = new File(indexDir, TIMEOUT_MAP);
+ if (!file.exists()) {
+ return;
+ }
+ try (FileInputStream in = new FileInputStream(file)) {
+ Properties prop = new Properties();
+ prop.load(in);
+ for(Entry<Object, Object> e : prop.entrySet()) {
+ timeoutMap.put(e.getKey().toString(), e.getValue().toString());
+ }
+ } catch (Exception e) {
+ log.warn("Could not load timeout map {} from {}",
+ TIMEOUT_MAP, indexDir, e);
+ }
+ }
+
+ private synchronized void storeTimeoutMap() {
+ if (indexDir == null || !indexDir.exists()) {
+ return;
+ }
+ File file = new File(indexDir, TIMEOUT_MAP);
+ try (FileOutputStream out = new FileOutputStream(file)) {
+ Properties prop = new Properties();
+ prop.putAll(timeoutMap);
+ prop.store(out, "Text extraction timed out for the following
binaries, and will not be retried");
+ } catch (Exception e) {
+ log.warn("Could not store timeout map {} from {}",
+ TIMEOUT_MAP, indexDir, e);
+ }
+ }
+
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ExtractedTextCache.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java?rev=1831979&r1=1831978&r2=1831979&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
Mon May 21 17:06:10 2018
@@ -282,7 +282,7 @@ public interface FulltextIndexConstants
String SUGGEST_ANALYZED = "suggestAnalyzed";
/**
- * Integer property indicating that LuceneIndex should be
+ * Integer property indicating that the index should be
* used in compat mode to specific version
*/
String COMPAT_MODE = "compatVersion";
@@ -311,7 +311,7 @@ public interface FulltextIndexConstants
String PROP_FUNCTION = "function";
/**
- * Boolean property which signal LuceneIndexEditor to refresh the stored
index definition
+ * Boolean property which signal FulltextIndexEditor to refresh the stored
index definition
*/
String PROP_REFRESH_DEFN = "refresh";
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java?rev=1831979&r1=1831978&r2=1831979&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
Mon May 21 17:06:10 2018
@@ -79,7 +79,6 @@ import static org.apache.jackrabbit.oak.
import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_DATA_CHILD_NAME;
import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.DECLARING_NODE_TYPES;
import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.ENTRY_COUNT_PROPERTY_NAME;
-import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEXING_MODE_NRT;
import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEXING_MODE_SYNC;
import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_COUNT;
import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.BLOB_SIZE;
@@ -154,7 +153,7 @@ public final class IndexDefinition imple
* This may not be the same time as when index was closed but the time of
checkpoint
* upto which index is upto date (OAK-6194)
*/
- static final String STATUS_LAST_UPDATED = "lastUpdated";
+ public static final String STATUS_LAST_UPDATED = "lastUpdated";
/**
* Meta property which provides the unique id
@@ -189,6 +188,8 @@ public final class IndexDefinition imple
private final NodeState root;
+ private final IndexFormatVersion version;
+
private final String funcName;
private final int blobSize;
@@ -276,6 +277,7 @@ public final class IndexDefinition imple
private final String indexPath;
private String uid;
private boolean reindexMode;
+ private IndexFormatVersion version;
public Builder(NodeState root, NodeState defn, String indexPath) {
this.root = checkNotNull(root);
@@ -288,12 +290,21 @@ public final class IndexDefinition imple
return this;
}
+ public Builder version(IndexFormatVersion version){
+ this.version = version;
+ return this;
+ }
+
+
public Builder reindex(){
this.reindexMode = true;
return this;
}
public IndexDefinition build(){
+ if (version == null){
+ version = determineIndexFormatVersion(defn);
+ }
if (uid == null){
uid = determineUniqueId(defn);
if (uid == null && !IndexDefinition.hasPersistedIndex(defn)){
@@ -305,16 +316,17 @@ public final class IndexDefinition imple
if (!reindexMode){
indexDefnStateToUse = getIndexDefinitionState(defn);
}
- return new IndexDefinition(root, indexDefnStateToUse, uid,
indexPath);
+ return new IndexDefinition(root, indexDefnStateToUse, version,
uid, indexPath);
}
}
public IndexDefinition(NodeState root, NodeState defn, String indexPath) {
- this(root, getIndexDefinitionState(defn), determineUniqueId(defn),
indexPath);
+ this(root, getIndexDefinitionState(defn),
determineIndexFormatVersion(defn), determineUniqueId(defn), indexPath);
}
- private IndexDefinition(NodeState root, NodeState defn, String uid, String
indexPath) {
+ private IndexDefinition(NodeState root, NodeState defn, IndexFormatVersion
version, String uid, String indexPath) {
this.root = root;
+ this.version = checkNotNull(version);
this.uid = uid;
this.definition = defn;
this.indexPath = checkNotNull(indexPath);
@@ -340,7 +352,7 @@ public final class IndexDefinition imple
String functionName = getOptionalValue(defn,
FulltextIndexConstants.FUNC_NAME, null);
if (fullTextEnabled && functionName == null){
- functionName = "lucene";
+ functionName = "fulltext";
}
this.funcName = functionName != null ? "native*" + functionName : null;
@@ -445,6 +457,11 @@ public final class IndexDefinition imple
return numOfDocs;
}
+ public IndexFormatVersion getVersion() {
+ return version;
+ }
+
+
public boolean isOfOldFormat(){
return !hasIndexingRules(definition);
}
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/NodeStateCloner.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/NodeStateCloner.java?rev=1831979&r1=1831978&r2=1831979&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/NodeStateCloner.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/NodeStateCloner.java
Mon May 21 17:06:10 2018
@@ -26,7 +26,7 @@ import org.apache.jackrabbit.oak.spi.sta
import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
-class NodeStateCloner {
+public class NodeStateCloner {
public static NodeState cloneVisibleState(NodeState state){
NodeBuilder builder = EMPTY_NODE.builder();
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/PropertyDefinition.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/PropertyDefinition.java?rev=1831979&r1=1831978&r2=1831979&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/PropertyDefinition.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/PropertyDefinition.java
Mon May 21 17:06:10 2018
@@ -68,17 +68,17 @@ public class PropertyDefinition {
final boolean isRegexp;
- final boolean index;
+ public final boolean index;
final boolean stored;
- final boolean nodeScopeIndex;
+ public final boolean nodeScopeIndex;
- final boolean propertyIndex;
+ public final boolean propertyIndex;
- final boolean analyzed;
+ public final boolean analyzed;
- final boolean ordered;
+ public final boolean ordered;
final boolean nullCheckEnabled;
@@ -86,17 +86,17 @@ public class PropertyDefinition {
final int includedPropertyTypes;
- final boolean relative;
+ public final boolean relative;
- final boolean useInSuggest;
+ public final boolean useInSuggest;
- final boolean useInSpellcheck;
+ public final boolean useInSpellcheck;
- final boolean facet;
+ public final boolean facet;
- final String[] ancestors;
+ public final String[] ancestors;
- final boolean excludeFromAggregate;
+ public final boolean excludeFromAggregate;
final int weight;
@@ -105,17 +105,17 @@ public class PropertyDefinition {
* its set to null
*/
@CheckForNull
- final String nonRelativeName;
+ public final String nonRelativeName;
/**
* For function-based indexes: the function name, in Polish notation.
*/
- final String function;
+ public final String function;
/**
* For function-based indexes: the function code, as tokens.
*/
- final String[] functionCode;
+ public final String[] functionCode;
public final ValuePattern valuePattern;
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ReindexOperations.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ReindexOperations.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ReindexOperations.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ReindexOperations.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search;
+
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+import static
org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE;
+import static
org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexEditorContext.configureUniqueId;
+
+public class ReindexOperations {
+ private final NodeState root;
+ private final NodeBuilder definitionBuilder;
+ private final String indexPath;
+
+ public ReindexOperations(NodeState root, NodeBuilder definitionBuilder,
String indexPath) {
+ this.root = root;
+ this.definitionBuilder = definitionBuilder;
+ this.indexPath = indexPath;
+ }
+
+ public IndexDefinition apply(boolean useStateFromBuilder) {
+ IndexFormatVersion version =
IndexDefinition.determineVersionForFreshIndex(definitionBuilder);
+ definitionBuilder.setProperty(IndexDefinition.INDEX_VERSION,
version.getVersion());
+
+ //Avoid obtaining the latest NodeState from builder as that would
force purge of current transient state
+ //as index definition does not get modified as part of IndexUpdate run
in most case we rely on base state
+ //For case where index definition is rewritten there we get fresh state
+ NodeState defnState = useStateFromBuilder ?
definitionBuilder.getNodeState() : definitionBuilder.getBaseState();
+ if (!IndexDefinition.isDisableStoredIndexDefinition()) {
+ definitionBuilder.setChildNode(INDEX_DEFINITION_NODE,
NodeStateCloner.cloneVisibleState(defnState));
+ }
+ String uid = configureUniqueId(definitionBuilder);
+
+ //Refresh the index definition based on update builder state
+ return IndexDefinition
+ .newBuilder(root, defnState, indexPath)
+ .version(version)
+ .uid(uid)
+ .reindex()
+ .build();
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/ReindexOperations.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/TextExtractionStatsMBean.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/TextExtractionStatsMBean.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/TextExtractionStatsMBean.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/TextExtractionStatsMBean.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search;
+
+import org.osgi.annotation.versioning.ProviderType;
+
+@ProviderType
+public interface TextExtractionStatsMBean {
+ String TYPE = "TextExtractionStats";
+
+ boolean isPreExtractedTextProviderConfigured();
+
+ boolean isAlwaysUsePreExtractedCache();
+
+ int getTextExtractionCount();
+
+ long getTotalTime();
+
+ int getPreFetchedCount();
+
+ String getExtractedTextSize();
+
+ String getBytesRead();
+
+ int getTimeoutCount();
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/TextExtractionStatsMBean.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/BlobByteSource.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/BlobByteSource.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/BlobByteSource.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/BlobByteSource.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search.spi.editor;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import com.google.common.io.ByteSource;
+import org.apache.jackrabbit.oak.api.Blob;
+
+public final class BlobByteSource extends ByteSource {
+ private final Blob blob;
+
+ public BlobByteSource(Blob blob) {
+ this.blob = blob;
+ }
+
+ @Override
+ public InputStream openStream() throws IOException {
+ return blob.getNewStream();
+ }
+
+ @Override
+ public long size() throws IOException {
+ return blob.length();
+ }
+
+ @Override
+ public boolean isEmpty() throws IOException {
+ return blob.length() == 0;
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/BlobByteSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/DocumentMaker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/DocumentMaker.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/DocumentMaker.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/DocumentMaker.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.search.spi.editor;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+/**
+ * A {@link DocumentMaker} is responsible for creating an instance of a
document {@link D} which depends on implementor implementation.
+ * For Apache Lucene that would be a Lucene {@code Document}, for Apache Solr
that might be a {@code SolrInputDocument}, etc.
+ */
+public interface DocumentMaker<D> {
+
+ /**
+ * create a document from the current state and list of modified properties
+ * @param state the node state
+ * @param isUpdate whether it is an update or not
+ * @param propertiesModified the list of modified properties
+ * @return a document to be indexed
+ * @throws IOException whether node state read operations or document
creation fail
+ */
+ D makeDocument(NodeState state, boolean isUpdate, List<PropertyState>
propertiesModified) throws IOException;
+
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/DocumentMaker.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextBinaryTextExtractor.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextBinaryTextExtractor.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextBinaryTextExtractor.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextBinaryTextExtractor.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,350 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.search.spi.editor;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeoutException;
+
+import javax.annotation.Nullable;
+
+import com.google.common.io.CountingInputStream;
+import org.apache.commons.io.IOUtils;
+import org.apache.jackrabbit.JcrConstants;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.io.LazyInputStream;
+import org.apache.jackrabbit.oak.plugins.index.fulltext.ExtractedText;
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.WriteOutContentHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.SAXException;
+
+import static org.apache.jackrabbit.JcrConstants.JCR_DATA;
+import static
org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexEditor.TEXT_EXTRACTION_ERROR;
+
+/**
+ *
+ */
+public class FulltextBinaryTextExtractor {
+
+ public static final String NO_NODEPATH_KEY = "*";
+
+ private static final Logger log =
LoggerFactory.getLogger(FulltextBinaryTextExtractor.class);
+ private static final Parser defaultParser = createDefaultParser();
+ private static final long SMALL_BINARY =
Long.getLong("oak.search.smallBinary", 16 * 1024);
+ private final TextExtractionStats textExtractionStats = new
TextExtractionStats();
+ private final ExtractedTextCache extractedTextCache;
+ private final IndexDefinition definition;
+ private final boolean reindex;
+ private Parser parser;
+ private TikaConfigHolder tikaConfig;
+ /**
+ * The media types supported by the parser used.
+ */
+ private Set<MediaType> supportedMediaTypes;
+ private Set<MediaType> nonIndexedMediaType;
+
+ public FulltextBinaryTextExtractor(ExtractedTextCache extractedTextCache,
IndexDefinition definition, boolean reindex) {
+ this.extractedTextCache = extractedTextCache;
+ this.definition = definition;
+ this.reindex = reindex;
+ }
+
+ public void done(boolean reindex){
+ textExtractionStats.log(reindex);
+ textExtractionStats.collectStats(extractedTextCache);
+ }
+
+ public Map<String, String> newBinary(
+ PropertyState property, NodeState state, String nodePath, String path) {
+ Map<String,String> fields = new HashMap<>();
+ Metadata metadata = new Metadata();
+
+ //jcr:mimeType is mandatory for a binary to be indexed
+ String type = state.getString(JcrConstants.JCR_MIMETYPE);
+ type = definition.getTikaMappedMimeType(type);
+
+ if (type == null || !isSupportedMediaType(type)) {
+ log.trace(
+ "[{}] Ignoring binary content for node {} due to unsupported (or
null) jcr:mimeType [{}]",
+ getIndexName(), path, type);
+ return fields;
+ }
+
+ metadata.set(Metadata.CONTENT_TYPE, type);
+ if (JCR_DATA.equals(property.getName())) {
+ String encoding = state.getString(JcrConstants.JCR_ENCODING);
+ if (encoding != null) { // not mandatory
+ metadata.set(Metadata.CONTENT_ENCODING, encoding);
+ }
+ }
+
+ for (Blob v : property.getValue(Type.BINARIES)) {
+ String value = parseStringValue(v, metadata, path, property.getName());
+ if (value == null){
+ continue;
+ }
+
+ if (nodePath != null){
+ fields.put(nodePath, value);
+ } else {
+ fields.put(NO_NODEPATH_KEY, value);
+ }
+ }
+ return fields;
+ }
+
+ private String parseStringValue(Blob v, Metadata metadata, String path,
String propertyName) {
+ String text = extractedTextCache.get(path, propertyName, v, reindex);
+ if (text == null){
+ text = parseStringValue0(v, metadata, path);
+ }
+ return text;
+ }
+
+ private String parseStringValue0(Blob v, Metadata metadata, String path) {
+ WriteOutContentHandler handler = new
WriteOutContentHandler(definition.getMaxExtractLength());
+ long start = System.currentTimeMillis();
+ long bytesRead = 0;
+ long length = v.length();
+ if (log.isDebugEnabled()) {
+ log.debug("Extracting {}, {} bytes, id {}", path, length,
v.getContentIdentity());
+ }
+ try {
+ CountingInputStream stream = new CountingInputStream(new
LazyInputStream(new BlobByteSource(v)));
+ try {
+ if (length > SMALL_BINARY) {
+ String name = "Extracting " + path + ", " + length + " bytes";
+ extractedTextCache.process(name, new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ getParser().parse(stream, handler, metadata, new ParseContext());
+ return null;
+ }
+ });
+ } else {
+ getParser().parse(stream, handler, metadata, new ParseContext());
+ }
+ } finally {
+ bytesRead = stream.getCount();
+ stream.close();
+ }
+ } catch (LinkageError e) {
+ // Capture errors caused by extraction libraries
+ // not being present. This is equivalent to disabling
+ // selected media types in configuration, so we can simply
+ // ignore these errors.
+ log.debug(
+ "[{}] Failed to extract text from a binary property: {}."
+ + " This often happens when some media types are disabled by
configuration."
+ + " The stack trace is included to flag some 'unintended'
failures",
+ getIndexName(), path, e);
+ extractedTextCache.put(v, ExtractedText.ERROR);
+ return TEXT_EXTRACTION_ERROR;
+ } catch (TimeoutException t) {
+ log.warn(
+ "[{}] Failed to extract text from a binary property due to timeout:
{}.",
+ getIndexName(), path);
+ extractedTextCache.put(v, ExtractedText.ERROR);
+ extractedTextCache.putTimeout(v, ExtractedText.ERROR);
+ return TEXT_EXTRACTION_ERROR;
+ } catch (Throwable t) {
+ // Capture and report any other full text extraction problems.
+ // The special STOP exception is used for normal termination.
+ if (!handler.isWriteLimitReached(t)) {
+ log.debug(
+ "[{}] Failed to extract text from a binary property: {}."
+ + " This is a fairly common case, and nothing to"
+ + " worry about. The stack trace is included to"
+ + " help improve the text extraction feature.",
+ getIndexName(), path, t);
+ extractedTextCache.put(v, ExtractedText.ERROR);
+ return TEXT_EXTRACTION_ERROR;
+ } else {
+ log.debug("Extracted text size exceeded configured limit({})",
definition.getMaxExtractLength());
+ }
+ }
+ String result = handler.toString();
+ if (bytesRead > 0) {
+ long time = System.currentTimeMillis() - start;
+ int len = result.length();
+ recordTextExtractionStats(time, bytesRead, len);
+ if (log.isDebugEnabled()) {
+ log.debug("Extracting {} took {} ms, {} bytes read, {} text size",
+ path, time, bytesRead, len);
+ }
+ }
+ extractedTextCache.put(v, new
ExtractedText(ExtractedText.ExtractionResult.SUCCESS, result));
+ return result;
+ }
+
+ private void recordTextExtractionStats(long timeInMillis, long bytesRead,
int textLength) {
+ textExtractionStats.addStats(timeInMillis, bytesRead, textLength);
+ }
+
+ private String getIndexName() {
+ return definition.getIndexName();
+ }
+
+ //~-------------------------------------------< Tika >
+
+ public TikaConfig getTikaConfig(){
+ if (tikaConfig == null) {
+ tikaConfig = initializeTikaConfig(definition);
+ }
+ return tikaConfig.config;
+ }
+
+ private Parser getParser() {
+ if (parser == null){
+ parser = initializeTikaParser(definition);
+ }
+ return parser;
+ }
+
+ private boolean isSupportedMediaType(String type) {
+ if (supportedMediaTypes == null) {
+ supportedMediaTypes = getParser().getSupportedTypes(new ParseContext());
+ nonIndexedMediaType = getNonIndexedMediaTypes();
+ }
+ MediaType mediaType = MediaType.parse(type);
+ return supportedMediaTypes.contains(mediaType) &&
!nonIndexedMediaType.contains(mediaType);
+ }
+
+ private Set<MediaType> getNonIndexedMediaTypes() {
+ InputStream configStream = null;
+ String configSource = null;
+ try {
+ if (definition.hasCustomTikaConfig()) {
+ configSource = String.format("Custom config at %s",
definition.getIndexPath());
+ configStream = definition.getTikaConfig();
+ } else {
+ URL configUrl =
FulltextIndexEditorContext.class.getResource("tika-config.xml");
+ configSource = "Default : tika-config.xml";
+ if (configUrl != null) {
+ configStream = configUrl.openStream();
+ }
+ }
+
+ if (configStream != null) {
+ return TikaParserConfig.getNonIndexedMediaTypes(configStream);
+ }
+ } catch (TikaException | IOException | SAXException e) {
+ log.warn("Tika configuration not available : " + configSource, e);
+ } finally {
+ IOUtils.closeQuietly(configStream);
+ }
+ return Collections.emptySet();
+ }
+
+
+ private static TikaConfigHolder initializeTikaConfig(@Nullable
IndexDefinition definition) {
+ ClassLoader current = Thread.currentThread().getContextClassLoader();
+ InputStream configStream = null;
+ String configSource = null;
+
+ try {
+
Thread.currentThread().setContextClassLoader(FulltextIndexEditorContext.class.getClassLoader());
+ if (definition != null && definition.hasCustomTikaConfig()) {
+ log.debug("[{}] Using custom tika config", definition.getIndexName());
+ configSource = "Custom config at " + definition.getIndexPath();
+ configStream = definition.getTikaConfig();
+ } else {
+ URL configUrl =
FulltextIndexEditorContext.class.getResource("tika-config.xml");
+ if (configUrl != null) {
+ configSource = configUrl.toString();
+ configStream = configUrl.openStream();
+ }
+ }
+
+ if (configStream != null) {
+ return new TikaConfigHolder(new TikaConfig(configStream),
configSource);
+ }
+ } catch (TikaException | IOException | SAXException e) {
+ log.warn("Tika configuration not available : " + configSource, e);
+ } finally {
+ IOUtils.closeQuietly(configStream);
+ Thread.currentThread().setContextClassLoader(current);
+ }
+ return new TikaConfigHolder(TikaConfig.getDefaultConfig(), "Default
Config");
+ }
+
+ private Parser initializeTikaParser(IndexDefinition definition) {
+ ClassLoader current = Thread.currentThread().getContextClassLoader();
+ try {
+ if (definition.hasCustomTikaConfig()) {
+
Thread.currentThread().setContextClassLoader(FulltextIndexEditorContext.class.getClassLoader());
+ return new AutoDetectParser(getTikaConfig());
+ }
+ } finally {
+ Thread.currentThread().setContextClassLoader(current);
+ }
+ return defaultParser;
+ }
+
+ private static AutoDetectParser createDefaultParser() {
+ ClassLoader current = Thread.currentThread().getContextClassLoader();
+ TikaConfigHolder configHolder = null;
+ try {
+ configHolder = initializeTikaConfig(null);
+
Thread.currentThread().setContextClassLoader(FulltextIndexEditorContext.class.getClassLoader());
+ log.info("Loaded default Tika Config from classpath {}", configHolder);
+ return new AutoDetectParser(configHolder.config);
+ } catch (Exception e) {
+ log.warn("Tika configuration not available : " + configHolder, e);
+ } finally {
+ Thread.currentThread().setContextClassLoader(current);
+ }
+ return new AutoDetectParser();
+ }
+
+ private static final class TikaConfigHolder{
+ final TikaConfig config;
+ final String sourceInfo;
+
+ public TikaConfigHolder(TikaConfig config, String sourceInfo) {
+ this.config = config;
+ this.sourceInfo = sourceInfo;
+ }
+
+ @Override
+ public String toString() {
+ return sourceInfo;
+ }
+ }
+
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextBinaryTextExtractor.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,558 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search.spi.editor;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.annotation.CheckForNull;
+import javax.jcr.PropertyType;
+
+import com.google.common.collect.Iterables;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+
+import org.apache.jackrabbit.oak.plugins.index.search.Aggregate;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition;
+import
org.apache.jackrabbit.oak.plugins.index.search.util.FunctionIndexProcessor;
+import org.apache.jackrabbit.oak.plugins.memory.StringPropertyState;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.jackrabbit.oak.commons.PathUtils.getName;
+
+import static
org.apache.jackrabbit.oak.plugins.index.search.util.ConfigUtil.getPrimaryTypeName;
+
+/**
+ * Abstract implementation of a {@link DocumentMaker}.
+ *
+ * @param <D> the type of documents to be indexed specific to subclasses
implementations
+ */
+public abstract class FulltextDocumentMaker<D> implements DocumentMaker<D> {
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ private FulltextBinaryTextExtractor textExtractor;
+ private IndexDefinition definition;
+ private IndexDefinition.IndexingRule indexingRule;
+ private String path;
+
+ abstract D initDoc();
+
+ abstract D finalizeDoc(D fields, boolean dirty, boolean facet);
+
+ abstract StringPropertyState createNodeNamePS();
+
+ abstract boolean isFacetingEnabled();
+
+ abstract boolean isNodeName(String pname);
+
+ protected abstract boolean indexTypeOrderedFields(String pname, int tag,
PropertyState property, PropertyDefinition pd);
+
+ protected abstract boolean addBinary(D doc, Map<String, String> binaryMap);
+
+ protected abstract boolean indexFacetProperty(D doc, int tag,
PropertyState property, String pname);
+
+ protected abstract boolean indexAnalyzedProperty(D doc, String pname,
String value, PropertyDefinition pd);
+
+ protected abstract boolean indexSuggestValue(D doc, String value);
+
+ protected abstract boolean indexSpellcheckValue(D doc, String value);
+
+ protected abstract boolean indexFulltextValue(D doc, String value);
+
+ protected abstract boolean indexTypedProperty(D doc, PropertyState
property, String pname, PropertyDefinition pd);
+
+ protected abstract boolean indexNotNullProperty(D doc, PropertyDefinition
pd);
+
+ protected abstract boolean indexNullProperty(D doc, PropertyDefinition pd);
+
+ protected abstract boolean indexAggregateValue(D doc,
Aggregate.NodeIncludeResult result, String value, PropertyDefinition pd);
+
+ protected abstract boolean indexNodeName(D doc, String value);
+
+ @CheckForNull
+ public D makeDocument(NodeState state) throws IOException {
+ return makeDocument(state, false,
Collections.<PropertyState>emptyList());
+ }
+
+ @CheckForNull
+ public D makeDocument(NodeState state, boolean isUpdate,
List<PropertyState> propertiesModified) throws IOException {
+ boolean facet = false;
+
+ D document = initDoc();
+ boolean dirty = false;
+
+ //We 'intentionally' are indexing node names only on root state as we
don't support indexing relative or
+ //regex for node name indexing
+ PropertyState nodenamePS = createNodeNamePS();
+ for (PropertyState property : Iterables.concat(state.getProperties(),
Collections.singleton(nodenamePS))) {
+ String pname = property.getName();
+
+ if (!isVisible(pname) && !isNodeName(pname)) {
+ continue;
+ }
+
+ PropertyDefinition pd = indexingRule.getConfig(pname);
+
+ if (pd == null || !pd.index){
+ continue;
+ }
+
+ if (pd.ordered) {
+ dirty |= addTypedOrderedFields(document, property, pname, pd);
+ }
+
+ dirty |= indexProperty(path, document, state, property, pname, pd);
+
+ facet |= pd.facet;
+ }
+
+ boolean[] dirties = indexAggregates(path, document, state);
+ dirty |= dirties[0]; // any (aggregate) indexing happened
+ facet |= dirties[1]; // facet indexing during (index-time) aggregation
+ dirty |= indexNullCheckEnabledProps(path, document, state);
+ dirty |= indexFunctionRestrictions(path, document, state);
+ dirty |= indexNotNullCheckEnabledProps(path, document, state);
+
+ dirty |= augmentCustomFields(path, document, state);
+
+ // Check if a node having a single property was modified/deleted
+ if (!dirty) {
+ dirty = indexIfSinglePropertyRemoved(propertiesModified);
+ }
+
+ if (isUpdate && !dirty) {
+ // updated the state but had no relevant changes
+ return null;
+ }
+
+ String name = getName(path);
+ if (indexingRule.isNodeNameIndexed()){
+ addNodeNameField(document, name);
+ dirty = true;
+ }
+
+ //For property index no use making an empty document if
+ //none of the properties are indexed
+ if(!indexingRule.indexesAllNodesOfMatchingType() && !dirty){
+ return null;
+ }
+
+ return finalizeDoc(document, dirty, facet);
+ }
+
+
+ private boolean indexFacets(D doc, PropertyState property, String pname,
PropertyDefinition pd) {
+ int tag = property.getType().tag();
+ int idxDefinedTag = pd.getType();
+ // Try converting type to the defined type in the index definition
+ if (tag != idxDefinedTag) {
+ log.debug("[{}] Facet property defined with type {} differs from
property {} with type {} in "
+ + "path {}",
+ getIndexName(),
+ Type.fromTag(idxDefinedTag, false), property.toString(),
+ Type.fromTag(tag, false), path);
+ tag = idxDefinedTag;
+ }
+ return indexFacetProperty(doc, tag, property, pname);
+ }
+
+ private boolean indexProperty(String path,
+ D doc,
+ NodeState state,
+ PropertyState property,
+ String pname,
+ PropertyDefinition pd) {
+ boolean includeTypeForFullText =
indexingRule.includePropertyType(property.getType().tag());
+
+ boolean dirty = false;
+ if (Type.BINARY.tag() == property.getType().tag()
+ && includeTypeForFullText) {
+ Map<String, String> binaryMap = newBinary(property, state, null,
path + "@" + pname);
+ addBinary(doc, binaryMap);
+ dirty = true;
+ } else {
+ if (pd.propertyIndex &&
pd.includePropertyType(property.getType().tag())) {
+ dirty |= addTypedFields(doc, property, pname, pd);
+ }
+
+ if (pd.fulltextEnabled() && includeTypeForFullText) {
+ for (String value : property.getValue(Type.STRINGS)) {
+
+ if (!includePropertyValue(value, pd)){
+ continue;
+ }
+
+ if (pd.analyzed &&
pd.includePropertyType(property.getType().tag())) {
+ indexAnalyzedProperty(doc, pname, value, pd);
+ }
+
+ if (pd.useInSuggest) {
+ indexSuggestValue(doc, value);
+ }
+
+ if (pd.useInSpellcheck) {
+ indexSpellcheckValue(doc, value);
+ }
+
+ if (pd.nodeScopeIndex) {
+ indexFulltextValue(doc, value);
+ }
+ dirty = true;
+ }
+ }
+ if (pd.facet && isFacetingEnabled()) {
+ dirty |= indexFacets(doc, property, pname, pd);
+ }
+
+ }
+
+ return dirty;
+ }
+
+ private boolean addTypedFields(D doc, PropertyState property, String
pname, PropertyDefinition pd) {
+ return indexTypedProperty(doc, property, pname, pd);
+ }
+
+ private boolean addTypedOrderedFields(D doc,
+ PropertyState property,
+ String pname,
+ PropertyDefinition pd) {
+ // Ignore and warn if property multi-valued as not supported
+ if (property.getType().isArray()) {
+ log.warn(
+ "[{}] Ignoring ordered property {} of type {} for path {}
as multivalued ordered property not supported",
+ getIndexName(), pname,
+ Type.fromTag(property.getType().tag(), true), path);
+ return false;
+ }
+
+ int tag = property.getType().tag();
+ int idxDefinedTag = pd.getType();
+ // Try converting type to the defined type in the index definition
+ if (tag != idxDefinedTag) {
+ log.debug(
+ "[{}] Ordered property defined with type {} differs from
property {} with type {} in "
+ + "path {}",
+ getIndexName(),
+ Type.fromTag(idxDefinedTag, false), property.toString(),
+ Type.fromTag(tag, false), path);
+ tag = idxDefinedTag;
+ }
+ return indexTypeOrderedFields(pname, tag, property, pd);
+ }
+
+ protected boolean includePropertyValue(PropertyState property, int i,
PropertyDefinition pd) {
+ if (property.getType().tag() == PropertyType.BINARY){
+ return true;
+ }
+
+ if (pd.valuePattern.matchesAll()) {
+ return true;
+ }
+
+ return includePropertyValue(property.getValue(Type.STRING, i), pd);
+ }
+
+ protected boolean includePropertyValue(String value, PropertyDefinition
pd){
+ return pd.valuePattern.matches(value);
+ }
+
+ private static boolean isVisible(String name) {
+ return name.charAt(0) != ':';
+ }
+
+ private Map<String,String> newBinary(
+ PropertyState property, NodeState state, String nodePath, String
path) {
+ if (textExtractor == null){
+ //Skip text extraction for sync indexing
+ return Collections.emptyMap();
+ }
+
+ return textExtractor.newBinary(property, state, nodePath, path);
+ }
+
+ private boolean augmentCustomFields(final String path, final D doc,
+ final NodeState document) {
+ boolean dirty = false;
+
+ // TODO : extract more generic SPI for augmentor factory
+
+// if (augmentorFactory != null) {
+// Iterable<Field> augmentedFields = augmentorFactory
+// .getIndexFieldProvider(indexingRule.getNodeTypeName())
+// .getAugmentedFields(path, document,
definition.getDefinitionNodeState());
+//
+// for (Field field : augmentedFields) {
+// fields.add(field);
+// dirty = true;
+// }
+// }
+
+ return dirty;
+ }
+
+ //~-------------------------------------------------------< NullCheck
Support >
+
+ private boolean indexNotNullCheckEnabledProps(String path, D doc,
NodeState state) {
+ boolean fieldAdded = false;
+ for (PropertyDefinition pd :
indexingRule.getNotNullCheckEnabledProperties()) {
+ if (isPropertyNotNull(state, pd)) {
+ fieldAdded = indexNotNullProperty(doc, pd);
+ }
+ }
+ return fieldAdded;
+ }
+
+
+ private boolean indexNullCheckEnabledProps(String path, D doc, NodeState
state) {
+ boolean fieldAdded = false;
+ for (PropertyDefinition pd :
indexingRule.getNullCheckEnabledProperties()) {
+ if (isPropertyNull(state, pd)) {
+ fieldAdded = indexNullProperty(doc, pd);
+ }
+ }
+ return fieldAdded;
+ }
+
+ private boolean indexFunctionRestrictions(String path, D fields, NodeState
state) {
+ boolean fieldAdded = false;
+ for (PropertyDefinition pd : indexingRule.getFunctionRestrictions()) {
+ PropertyState functionValue = calculateValue(path, state,
pd.functionCode);
+ if (functionValue != null) {
+ if (pd.ordered) {
+ addTypedOrderedFields(fields, functionValue, pd.function,
pd);
+ }
+ addTypedFields(fields, functionValue, pd.function, pd);
+ fieldAdded = true;
+ }
+ }
+ return fieldAdded;
+ }
+
+ private PropertyState calculateValue(String path, NodeState state,
String[] functionCode) {
+ try {
+ return FunctionIndexProcessor.tryCalculateValue(path, state,
functionCode);
+ } catch (RuntimeException e) {
+ log.error("Failed to calculate function value for {} at {}",
+ Arrays.toString(functionCode), path, e);
+ throw e;
+ }
+ }
+
+ private boolean indexIfSinglePropertyRemoved(List<PropertyState>
propertiesModified) {
+ boolean dirty = false;
+ for (PropertyState ps : propertiesModified) {
+ PropertyDefinition pd = indexingRule.getConfig(ps.getName());
+ if (pd != null
+ && pd.index
+ && (pd.includePropertyType(ps.getType().tag())
+ ||
indexingRule.includePropertyType(ps.getType().tag()))) {
+ dirty = true;
+ break;
+ }
+ }
+ return dirty;
+ }
+
+ /**
+ * Determine if the property as defined by PropertyDefinition exists or
not.
+ *
+ * <p>For relative property if the intermediate nodes do not exist then
property is
+ * <bold>not</bold> considered to be null</p>
+ *
+ * @return true if the property does not exist
+ */
+ private boolean isPropertyNull(NodeState state, PropertyDefinition pd){
+ NodeState propertyNode = getPropertyNode(state, pd);
+ if (!propertyNode.exists()){
+ return false;
+ }
+ return !propertyNode.hasProperty(pd.nonRelativeName);
+ }
+
+ /**
+ * Determine if the property as defined by PropertyDefinition exists or
not.
+ *
+ * <p>For relative property if the intermediate nodes do not exist then
property is
+ * considered to be null</p>
+ *
+ * @return true if the property exists
+ */
+ private boolean isPropertyNotNull(NodeState state, PropertyDefinition pd){
+ NodeState propertyNode = getPropertyNode(state, pd);
+ if (!propertyNode.exists()){
+ return false;
+ }
+ return propertyNode.hasProperty(pd.nonRelativeName);
+ }
+
+ private static NodeState getPropertyNode(NodeState nodeState,
PropertyDefinition pd) {
+ if (!pd.relative){
+ return nodeState;
+ }
+ NodeState node = nodeState;
+ for (String name : pd.ancestors) {
+ node = node.getChildNode(name);
+ }
+ return node;
+ }
+
+ /**
+ * index aggregates on a certain path
+ * @param path the path of the node
+ * @param fields the list of fields
+ * @param state the node state
+ * @return an array of booleans whose first element is {@code true} if any
indexing has happened
+ * and the second element is {@code true} if facets on any (aggregate)
property have been indexed
+ */
+ private boolean[] indexAggregates(final String path, final D fields,
+ final NodeState state) {
+ final AtomicBoolean dirtyFlag = new AtomicBoolean();
+ final AtomicBoolean facetFlag = new AtomicBoolean();
+ indexingRule.getAggregate().collectAggregates(state, new
Aggregate.ResultCollector() {
+ @Override
+ public void onResult(Aggregate.NodeIncludeResult result) {
+ boolean dirty = indexAggregatedNode(path, fields, result);
+ if (dirty) {
+ dirtyFlag.set(true);
+ }
+ }
+
+ @Override
+ public void onResult(Aggregate.PropertyIncludeResult result) {
+ boolean dirty = false;
+ if (result.pd.ordered) {
+ dirty |= addTypedOrderedFields(fields,
result.propertyState,
+ result.propertyPath, result.pd);
+ }
+ dirty |= indexProperty(path, fields, state,
result.propertyState,
+ result.propertyPath, result.pd);
+
+ if (result.pd.facet) {
+ facetFlag.set(true);
+ }
+ if (dirty) {
+ dirtyFlag.set(true);
+ }
+ }
+ });
+ return new boolean[]{dirtyFlag.get(), facetFlag.get()};
+ }
+ /**
+ * Create the fulltext field from the aggregated nodes. If result is for
aggregate for a relative node
+ * include then
+ * @param path current node path
+ * @param doc document
+ * @param result aggregate result
+ * @return true if a field was created for passed node result
+ */
+ private boolean indexAggregatedNode(String path, D doc,
Aggregate.NodeIncludeResult result) {
+ //rule for node being aggregated might be null if such nodes
+ //are not indexed on there own. In such cases we rely in current
+ //rule for some checks
+ IndexDefinition.IndexingRule ruleAggNode = definition
+
.getApplicableIndexingRule(getPrimaryTypeName(result.nodeState));
+ boolean dirty = false;
+
+ for (PropertyState property : result.nodeState.getProperties()){
+ String pname = property.getName();
+ String propertyPath = PathUtils.concat(result.nodePath, pname);
+
+ if (!isVisible(pname)) {
+ continue;
+ }
+
+ //Check if type is indexed
+ int type = property.getType().tag();
+ if (ruleAggNode != null ) {
+ if (!ruleAggNode.includePropertyType(type)) {
+ continue;
+ }
+ } else if (!indexingRule.includePropertyType(type)){
+ continue;
+ }
+
+ //Check if any explicit property defn is defined via relative path
+ // and is marked to exclude this property from being indexed. We
exclude
+ //it from aggregation if
+ // 1. Its not to be indexed i.e. index=false
+ // 2. Its explicitly excluded from aggregation i.e.
excludeFromAggregation=true
+ PropertyDefinition pdForRootNode =
indexingRule.getConfig(propertyPath);
+ if (pdForRootNode != null && (!pdForRootNode.index ||
pdForRootNode.excludeFromAggregate)) {
+ continue;
+ }
+
+ if (Type.BINARY == property.getType()) {
+ String aggreagtedNodePath = PathUtils.concat(path,
result.nodePath);
+ //Here the fulltext is being created for aggregate root hence
nodePath passed
+ //should be null
+ String nodePath = result.isRelativeNode() ?
result.rootIncludePath : null;
+ Map<String, String> stringStringMap = newBinary(property,
result.nodeState, nodePath, aggreagtedNodePath + "@" + pname);
+ addBinary(doc, stringStringMap);
+ dirty = true;
+ } else {
+ PropertyDefinition pd = null;
+ if (ruleAggNode != null){
+ pd = ruleAggNode.getConfig(pname);
+ }
+
+ if (pd != null && !pd.nodeScopeIndex){
+ continue;
+ }
+
+ for (String value : property.getValue(Type.STRINGS)) {
+ dirty = indexAggregateValue(doc, result, value, pd);
+ }
+ }
+ }
+ return dirty;
+ }
+
+ private String getIndexName() {
+ return definition.getIndexName();
+ }
+
+ /**
+ * Extracts the local name of the current node ignoring any namespace
prefix
+ *
+ * @param name node name
+ */
+ private void addNodeNameField(D doc, String name) {
+ //TODO Need to check if it covers all cases
+ int colon = name.indexOf(':');
+ String value = colon < 0 ? name : name.substring(colon + 1);
+
+ //For now just add a single term. Later we can look into using
different analyzer
+ //to analyze the node name and add multiple terms. Like add multiple
terms for a
+ //cameCase file name to allow faster like search
+ indexNodeName(doc, value);
+ }
+
+
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java
------------------------------------------------------------------------------
svn:eol-style = native
