Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.search.spi.editor;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.index.IndexEditor;
+import org.apache.jackrabbit.oak.plugins.index.search.Aggregate;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.PropertyUpdateCallback;
+import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
+import org.apache.jackrabbit.oak.spi.commit.Editor;
+import org.apache.jackrabbit.oak.spi.filter.PathFilter;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
+
+/**
+ * Abstract implementation of an {@link IndexEditor} which supports index time
aggregation.
+ */
+public class FulltextIndexEditor<D> implements IndexEditor,
Aggregate.AggregateRoot {
+
+ private static final Logger log =
+ LoggerFactory.getLogger(FulltextIndexEditor.class);
+
+ public static final String TEXT_EXTRACTION_ERROR = "TextExtractionError";
+
+ private final FulltextIndexEditorContext<D> context;
+
+ /** Name of this node, or {@code null} for the root node. */
+ private final String name;
+
+ /** Parent editor or {@code null} if this is the root editor. */
+ private final FulltextIndexEditor parent;
+
+ /** Path of this editor, built lazily in {@link #getPath()}. */
+ private String path;
+
+ private boolean propertiesChanged = false;
+
+ private List<PropertyState> propertiesModified = Lists.newArrayList();
+
+ /**
+ * Flag indicating if the current tree being traversed has a deleted parent.
+ */
+ private final boolean isDeleted;
+
+ private IndexDefinition.IndexingRule indexingRule;
+
+ private List<Aggregate.Matcher> currentMatchers = Collections.emptyList();
+
+ private final MatcherState matcherState;
+
+ private final PathFilter.Result pathFilterResult;
+
+ FulltextIndexEditor(FulltextIndexEditorContext<D> context) throws
CommitFailedException {
+ this.parent = null;
+ this.name = null;
+ this.path = "/";
+ this.context = context;
+ this.isDeleted = false;
+ this.matcherState = MatcherState.NONE;
+ this.pathFilterResult =
context.getDefinition().getPathFilter().filter(PathUtils.ROOT_PATH);
+ }
+
+ private FulltextIndexEditor(FulltextIndexEditor<D> parent, String name,
+ MatcherState matcherState,
+ PathFilter.Result pathFilterResult,
+ boolean isDeleted) {
+ this.parent = parent;
+ this.name = name;
+ this.path = null;
+ this.context = parent.context;
+ this.isDeleted = isDeleted;
+ this.matcherState = matcherState;
+ this.pathFilterResult = pathFilterResult;
+ }
+
+ public String getPath() {
+ if (path == null) { // => parent != null
+ path = concat(parent.getPath(), name);
+ }
+ return path;
+ }
+
+ @Override
+ public void enter(NodeState before, NodeState after) {
+ if (EmptyNodeState.MISSING_NODE == before && parent == null){
+ context.enableReindexMode();
+ }
+
+ //Only check for indexing if the result is include.
+ //In case like TRAVERSE nothing needs to be indexed for those
+ //path
+ if (pathFilterResult == PathFilter.Result.INCLUDE) {
+ //For traversal in deleted sub tree before state has to be used
+ NodeState current = after.exists() ? after : before;
+ indexingRule = getDefinition().getApplicableIndexingRule(current);
+
+ if (indexingRule != null) {
+ currentMatchers = indexingRule.getAggregate().createMatchers(this);
+ }
+ }
+ }
+
+ @Override
+ public void leave(NodeState before, NodeState after)
+ throws CommitFailedException {
+ if (propertiesChanged || !before.exists()) {
+ String path = getPath();
+ if (addOrUpdate(path, after, before.exists())) {
+ long indexed = context.incIndexedNodes();
+ if (indexed % 1000 == 0) {
+ log.debug("[{}] => Indexed {} nodes...", getIndexName(), indexed);
+ }
+ }
+ }
+
+ for (Aggregate.Matcher m : matcherState.affectedMatchers){
+ m.markRootDirty();
+ }
+
+ if (parent == null) {
+ PropertyUpdateCallback callback = context.getPropertyUpdateCallback();
+ if (callback != null) {
+ callback.done();
+ }
+
+ try {
+ context.closeWriter();
+ } catch (IOException e) {
+ CommitFailedException ce = new CommitFailedException("Fulltext", 4,
+ "Failed to close the Fulltext index " +
context.getIndexingContext().getIndexPath(), e);
+ context.getIndexingContext().indexUpdateFailed(ce);
+ throw ce;
+ }
+ if (context.getIndexedNodes() > 0) {
+ log.debug("[{}] => Indexed {} nodes, done.", getIndexName(),
context.getIndexedNodes());
+ }
+ }
+ }
+
+ @Override
+ public void propertyAdded(PropertyState after) {
+ markPropertyChanged(after.getName());
+ checkAggregates(after.getName());
+ propertyUpdated(null, after);
+ }
+
+ @Override
+ public void propertyChanged(PropertyState before, PropertyState after) {
+ markPropertyChanged(before.getName());
+ propertiesModified.add(before);
+ checkAggregates(before.getName());
+ propertyUpdated(before, after);
+ }
+
+ @Override
+ public void propertyDeleted(PropertyState before) {
+ markPropertyChanged(before.getName());
+ propertiesModified.add(before);
+ checkAggregates(before.getName());
+ propertyUpdated(before, null);
+ }
+
+ @Override
+ public Editor childNodeAdded(String name, NodeState after) {
+ PathFilter.Result filterResult = getPathFilterResult(name);
+ if (filterResult != PathFilter.Result.EXCLUDE) {
+ return new FulltextIndexEditor(this, name, getMatcherState(name, after),
filterResult, false);
+ }
+ return null;
+ }
+
+ @Override
+ public Editor childNodeChanged(
+ String name, NodeState before, NodeState after) {
+ PathFilter.Result filterResult = getPathFilterResult(name);
+ if (filterResult != PathFilter.Result.EXCLUDE) {
+ return new FulltextIndexEditor(this, name, getMatcherState(name, after),
filterResult, false);
+ }
+ return null;
+ }
+
+ @Override
+ public Editor childNodeDeleted(String name, NodeState before)
+ throws CommitFailedException {
+ PathFilter.Result filterResult = getPathFilterResult(name);
+ if (filterResult == PathFilter.Result.EXCLUDE) {
+ return null;
+ }
+
+ if (!isDeleted) {
+ // tree deletion is handled on the parent node
+ String path = concat(getPath(), name);
+ try {
+ FulltextIndexWriter writer = context.getWriter();
+ // Remove all index entries in the removed subtree
+ writer.deleteDocuments(path);
+ this.context.indexUpdate();
+ } catch (IOException e) {
+ CommitFailedException ce = new CommitFailedException("Fulltext", 5,
"Failed to remove the index entries of"
+ + " the removed subtree " + path + "for index " +
context.getIndexingContext().getIndexPath(), e);
+ context.getIndexingContext().indexUpdateFailed(ce);
+ throw ce;
+ }
+ }
+
+ MatcherState ms = getMatcherState(name, before);
+ if (!ms.isEmpty()){
+ return new FulltextIndexEditor(this, name, ms, filterResult, true);
+ }
+ return null; // no need to recurse down the removed subtree
+ }
+
+ FulltextIndexEditorContext<D> getContext() {
+ return context;
+ }
+
+ private boolean addOrUpdate(String path, NodeState state, boolean isUpdate)
+ throws CommitFailedException {
+ try {
+ D d = makeDocument(path, state, isUpdate);
+ if (d != null) {
+ if (log.isTraceEnabled()) {
+ log.trace("[{}] Indexed document for {} is {}", getIndexName(),
path, d);
+ }
+ context.indexUpdate();
+ context.getWriter().updateDocument(path, d);
+ return true;
+ }
+ } catch (IOException e) {
+ CommitFailedException ce = new CommitFailedException("Fulltext", 3,
+ "Failed to index the node " + path, e);
+ context.getIndexingContext().indexUpdateFailed(ce);
+ throw ce;
+ } catch (IllegalArgumentException ie) {
+ log.warn("Failed to index the node [{}]", path, ie);
+ }
+ return false;
+ }
+
+ private D makeDocument(String path, NodeState state, boolean isUpdate)
throws IOException {
+ if (!isIndexable()) {
+ return null;
+ }
+ return context.newDocumentMaker(indexingRule, path).makeDocument(state,
isUpdate, propertiesModified);
+ }
+
+
+ //~-------------------------------------------------------< Aggregate >
+
+ @Override
+ public void markDirty() {
+ propertiesChanged = true;
+ }
+
+ private MatcherState getMatcherState(String name, NodeState after) {
+ List<Aggregate.Matcher> matched = Lists.newArrayList();
+ List<Aggregate.Matcher> inherited = Lists.newArrayList();
+ for (Aggregate.Matcher m : Iterables.concat(matcherState.inherited,
currentMatchers)) {
+ Aggregate.Matcher result = m.match(name, after);
+ if (result.getStatus() == Aggregate.Matcher.Status.MATCH_FOUND){
+ matched.add(result);
+ }
+
+ if (result.getStatus() != Aggregate.Matcher.Status.FAIL){
+ inherited.addAll(result.nextSet());
+ }
+ }
+
+ if (!matched.isEmpty() || !inherited.isEmpty()) {
+ return new MatcherState(matched, inherited);
+ }
+ return MatcherState.NONE;
+ }
+
+
+ /**
+ * Determines which all matchers are affected by this property change
+ *
+ * @param name modified property name
+ */
+ private void checkAggregates(String name) {
+ for (Aggregate.Matcher m : matcherState.matched) {
+ if (!matcherState.affectedMatchers.contains(m)
+ && m.aggregatesProperty(name)) {
+ matcherState.affectedMatchers.add(m);
+ }
+ }
+ }
+
+ static class MatcherState {
+ final static MatcherState NONE = new
MatcherState(Collections.<Aggregate.Matcher>emptyList(),
+ Collections.<Aggregate.Matcher>emptyList());
+
+ final List<Aggregate.Matcher> matched;
+ final List<Aggregate.Matcher> inherited;
+ final Set<Aggregate.Matcher> affectedMatchers;
+
+ public MatcherState(List<Aggregate.Matcher> matched,
+ List<Aggregate.Matcher> inherited){
+ this.matched = matched;
+ this.inherited = inherited;
+
+ //Affected matches would only be used when there are
+ //some matched matchers
+ if (matched.isEmpty()){
+ affectedMatchers = Collections.emptySet();
+ } else {
+ affectedMatchers = Sets.newIdentityHashSet();
+ }
+ }
+
+ public boolean isEmpty() {
+ return matched.isEmpty() && inherited.isEmpty();
+ }
+ }
+
+ private void markPropertyChanged(String name) {
+ if (isIndexable()
+ && !propertiesChanged
+ && indexingRule.isIndexed(name)) {
+ propertiesChanged = true;
+ }
+ }
+
+ private void propertyUpdated(PropertyState before, PropertyState after) {
+ PropertyUpdateCallback callback = context.getPropertyUpdateCallback();
+
+ //Avoid further work if no callback is present
+ if (callback == null) {
+ return;
+ }
+
+ String propertyName = before != null ? before.getName() : after.getName();
+
+ if (isIndexable()) {
+ PropertyDefinition pd = indexingRule.getConfig(propertyName);
+ if (pd != null) {
+ callback.propertyUpdated(getPath(), propertyName, pd, before, after);
+ }
+ }
+
+ for (Aggregate.Matcher m : matcherState.matched) {
+ if (m.aggregatesProperty(propertyName)) {
+ Aggregate.Include i = m.getCurrentInclude();
+ if (i instanceof Aggregate.PropertyInclude) {
+ PropertyDefinition pd = ((Aggregate.PropertyInclude)
i).getPropertyDefinition();
+ String propertyRelativePath = PathUtils.concat(m.getMatchedPath(),
propertyName);
+
+ callback.propertyUpdated(m.getRootPath(), propertyRelativePath, pd,
before, after);
+ }
+ }
+ }
+ }
+
+ private IndexDefinition getDefinition() {
+ return context.getDefinition();
+ }
+
+ private boolean isIndexable(){
+ return indexingRule != null;
+ }
+
+ private PathFilter.Result getPathFilterResult(String childNodeName) {
+ return context.getDefinition().getPathFilter().filter(concat(getPath(),
childNodeName));
+ }
+
+ private String getIndexName() {
+ return context.getDefinition().getIndexName();
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.search.spi.editor;
+
+import java.io.IOException;
+import java.util.Calendar;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PerfLogger;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
+import org.apache.jackrabbit.oak.plugins.index.IndexingContext;
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.NodeStateCloner;
+import org.apache.jackrabbit.oak.plugins.index.search.PropertyUpdateCallback;
+import org.apache.jackrabbit.oak.plugins.index.search.ReindexOperations;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.apache.jackrabbit.util.ISO8601;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_REFRESH_DEFN;
+import static
org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE;
+
+/**
+ *
+ */
+public abstract class FulltextIndexEditorContext<D> {
+ private static final Logger log = LoggerFactory
+ .getLogger(FulltextIndexEditorContext.class);
+
+ private static final PerfLogger PERF_LOGGER =
+ new
PerfLogger(LoggerFactory.getLogger(FulltextIndexEditorContext.class.getName() +
".perf"));
+
+ private IndexDefinition definition;
+
+ private final NodeBuilder definitionBuilder;
+
+ private final FulltextIndexWriterFactory indexWriterFactory;
+
+ private FulltextIndexWriter writer = null;
+
+ private long indexedNodes;
+
+ private final IndexUpdateCallback updateCallback;
+
+ private boolean reindex;
+
+ private final ExtractedTextCache extractedTextCache;
+
+ private final NodeState root;
+
+ private final IndexingContext indexingContext;
+
+ private final boolean asyncIndexing;
+
+ //Intentionally static, so that it can be set without passing around clock
objects
+ //Set for testing ONLY
+ private static Clock clock = Clock.SIMPLE;
+
+ private final boolean indexDefnRewritten;
+
+ private FulltextBinaryTextExtractor textExtractor;
+
+ private PropertyUpdateCallback propertyUpdateCallback;
+
+ FulltextIndexEditorContext(NodeState root, NodeBuilder definition,
+ @Nullable IndexDefinition indexDefinition,
+ IndexUpdateCallback updateCallback,
+ FulltextIndexWriterFactory indexWriterFactory,
+ ExtractedTextCache extractedTextCache,
+ IndexingContext indexingContext, boolean
asyncIndexing) {
+ this.root = root;
+ this.indexingContext = checkNotNull(indexingContext);
+ this.definitionBuilder = definition;
+ this.indexWriterFactory = indexWriterFactory;
+ this.definition = indexDefinition != null ? indexDefinition :
+ createIndexDefinition(root, definition, indexingContext,
asyncIndexing);
+ this.indexedNodes = 0;
+ this.updateCallback = updateCallback;
+ this.extractedTextCache = extractedTextCache;
+ this.asyncIndexing = asyncIndexing;
+ if (this.definition.isOfOldFormat()){
+ indexDefnRewritten = true;
+ IndexDefinition.updateDefinition(definition,
indexingContext.getIndexPath());
+ } else {
+ indexDefnRewritten = false;
+ }
+ }
+
+
+ abstract DocumentMaker<D> newDocumentMaker(IndexDefinition.IndexingRule
rule, String path);
+
+ abstract FulltextBinaryTextExtractor
createBinaryTextExtractor(ExtractedTextCache extractedTextCache,
IndexDefinition definition, boolean reindex);
+
+ FulltextIndexWriter getWriter() throws IOException {
+ if (writer == null) {
+ //Lazy initialization so as to ensure that definition is based
+ //on latest NodeBuilder state specially in case of reindexing
+ writer = indexWriterFactory.newInstance(definition, definitionBuilder,
reindex);
+ }
+ return writer;
+ }
+
+ public IndexingContext getIndexingContext() {
+ return indexingContext;
+ }
+
+ @CheckForNull
+ public PropertyUpdateCallback getPropertyUpdateCallback() {
+ return propertyUpdateCallback;
+ }
+
+ void setPropertyUpdateCallback(PropertyUpdateCallback
propertyUpdateCallback) {
+ this.propertyUpdateCallback = propertyUpdateCallback;
+ }
+
+ /**
+ * close writer if it's not null
+ */
+ void closeWriter() throws IOException {
+ Calendar currentTime = getCalendar();
+ final long start = PERF_LOGGER.start();
+ boolean indexUpdated = getWriter().close(currentTime.getTimeInMillis());
+
+ if (indexUpdated) {
+ PERF_LOGGER.end(start, -1, "Closed writer for directory {}", definition);
+ //OAK-2029 Record the last updated status so
+ //as to make IndexTracker detect changes when index
+ //is stored in file system
+ NodeBuilder status =
definitionBuilder.child(IndexDefinition.STATUS_NODE);
+ status.setProperty(IndexDefinition.STATUS_LAST_UPDATED,
getUpdatedTime(currentTime), Type.DATE);
+ status.setProperty("indexedNodes", indexedNodes);
+
+ PERF_LOGGER.end(start, -1, "Overall Closed IndexWriter for directory
{}", definition);
+
+ if (textExtractor != null){
+ textExtractor.done(reindex);
+ }
+ }
+ }
+
+ private String getUpdatedTime(Calendar currentTime) {
+ CommitInfo info = getIndexingContext().getCommitInfo();
+ String checkpointTime = (String)
info.getInfo().get(IndexConstants.CHECKPOINT_CREATION_TIME);
+ if (checkpointTime != null) {
+ return checkpointTime;
+ }
+ return ISO8601.format(currentTime);
+ }
+
+ /** Only set for testing */
+ static void setClock(Clock c) {
+ checkNotNull(c);
+ clock = c;
+ }
+
+ static private Calendar getCalendar() {
+ Calendar ret = Calendar.getInstance();
+ ret.setTime(clock.getDate());
+ return ret;
+ }
+
+ public void enableReindexMode(){
+ reindex = true;
+ ReindexOperations reindexOps = new ReindexOperations(root,
definitionBuilder, definition.getIndexPath());
+ definition = reindexOps.apply(indexDefnRewritten);
+ }
+
+ public long incIndexedNodes() {
+ indexedNodes++;
+ return indexedNodes;
+ }
+
+ boolean isAsyncIndexing() {
+ return asyncIndexing;
+ }
+
+ public long getIndexedNodes() {
+ return indexedNodes;
+ }
+
+ void indexUpdate() throws CommitFailedException {
+ updateCallback.indexUpdate();
+ }
+
+ public IndexDefinition getDefinition() {
+ return definition;
+ }
+
+ private FulltextBinaryTextExtractor getTextExtractor(){
+ if (textExtractor == null && isAsyncIndexing()){
+ //Create lazily to ensure that if its reindex case then update
definition is picked
+ textExtractor = createBinaryTextExtractor(extractedTextCache,
definition, reindex);
+ }
+ return textExtractor;
+ }
+
+ public boolean isReindex() {
+ return reindex;
+ }
+
+ public static String configureUniqueId(NodeBuilder definition) {
+ NodeBuilder status = definition.child(IndexDefinition.STATUS_NODE);
+ String uid = status.getString(IndexDefinition.PROP_UID);
+ if (uid == null) {
+ try {
+ uid = String.valueOf(Clock.SIMPLE.getTimeIncreasing());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ uid = String.valueOf(Clock.SIMPLE.getTime());
+ }
+ status.setProperty(IndexDefinition.PROP_UID, uid);
+ }
+ return uid;
+ }
+
+ private static IndexDefinition createIndexDefinition(NodeState root,
NodeBuilder definition, IndexingContext
+ indexingContext, boolean asyncIndexing) {
+ NodeState defnState = definition.getBaseState();
+ if (asyncIndexing && !IndexDefinition.isDisableStoredIndexDefinition()){
+ if (definition.getBoolean(PROP_REFRESH_DEFN)){
+ definition.removeProperty(PROP_REFRESH_DEFN);
+ NodeState clonedState = NodeStateCloner.cloneVisibleState(defnState);
+ definition.setChildNode(INDEX_DEFINITION_NODE, clonedState);
+ log.info("Refreshed the index definition for [{}]",
indexingContext.getIndexPath());
+ if (log.isDebugEnabled()){
+ log.debug("Updated index definition is {}",
NodeStateUtils.toString(clonedState));
+ }
+ } else if (!definition.hasChildNode(INDEX_DEFINITION_NODE)){
+ definition.setChildNode(INDEX_DEFINITION_NODE,
NodeStateCloner.cloneVisibleState(defnState));
+ log.info("Stored the cloned index definition for [{}]. Changes in
index definition would now only be " +
+ "effective post reindexing", indexingContext.getIndexPath());
+ }
+ }
+ return new IndexDefinition(root, defnState,indexingContext.getIndexPath());
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriter.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriter.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriter.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search.spi.editor;
+
+import java.io.IOException;
+
+public interface FulltextIndexWriter<D> {
+
+ /**
+ * Updates the document having given path
+ *
+ * @param path path of the NodeState which the Document represents
+ * @param doc updated document
+ */
+ void updateDocument(String path, D doc) throws IOException;
+
+ /**
+ * Deletes documents which are same or child of given path
+ *
+ * @param path path whose children need to be deleted
+ */
+ void deleteDocuments(String path) throws IOException;
+
+ /**
+ * Closes the underlying resources.
+ *
+ * @param timestamp timestamp to be used for recording at status in
NodeBuilder
+ * @return true if index was updated or any write happened.
+ */
+ boolean close(long timestamp) throws IOException;
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriter.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriterFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriterFactory.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriterFactory.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriterFactory.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search.spi.editor;
+
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+
+/**
+ * Factory class for {@link FulltextIndexWriter}s
+ */
+public interface FulltextIndexWriterFactory {
+
+ FulltextIndexWriter newInstance(IndexDefinition definition, NodeBuilder
definitionBuilder, boolean reindex);
+
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriterFactory.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TextExtractionStats.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TextExtractionStats.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TextExtractionStats.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TextExtractionStats.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search.spi.editor;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
+
+class TextExtractionStats {
+ private static final Logger log =
LoggerFactory.getLogger(TextExtractionStats.class);
+ /**
+ * Log stats only if time spent is more than 1 min
+ */
+ private static final long LOGGING_THRESHOLD = TimeUnit.MINUTES.toMillis(1);
+ private int count;
+ private long totalBytesRead;
+ private long totalTime;
+ private long totalTextLength;
+
+ public void addStats(long timeInMillis, long bytesRead, int textLength) {
+ count++;
+ totalBytesRead += bytesRead;
+ totalTime += timeInMillis;
+ totalTextLength += textLength;
+ }
+
+ public void log(boolean reindex) {
+ if (log.isDebugEnabled()) {
+ log.debug("Text extraction stats {}", this);
+ } else if (anyParsingDone() && (reindex || isTakingLotsOfTime())) {
+ log.info("Text extraction stats {}", this);
+ }
+ }
+
+ public void collectStats(ExtractedTextCache cache){
+ cache.addStats(count, totalTime, totalBytesRead, totalTextLength);
+ }
+
+ private boolean isTakingLotsOfTime() {
+ return totalTime > LOGGING_THRESHOLD;
+ }
+
+ private boolean anyParsingDone() {
+ return count > 0;
+ }
+
+ @Override
+ public String toString() {
+ return String.format(" %d (Time Taken %s, Bytes Read %s, Extracted
text size %s)",
+ count,
+ timeInWords(totalTime),
+ humanReadableByteCount(totalBytesRead),
+ humanReadableByteCount(totalTextLength));
+ }
+
+ private static String timeInWords(long millis) {
+ return String.format("%d min, %d sec",
+ TimeUnit.MILLISECONDS.toMinutes(millis),
+ TimeUnit.MILLISECONDS.toSeconds(millis) -
+
TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis))
+ );
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TextExtractionStats.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TikaParserConfig.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TikaParserConfig.java?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TikaParserConfig.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TikaParserConfig.java
Mon May 21 17:06:10 2018
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search.spi.editor;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.xml.parsers.DocumentBuilder;
+
+import com.google.common.base.Strings;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.ParseContext;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+public class TikaParserConfig {
+ private static final String EMPTY_PARSER =
"org.apache.tika.parser.EmptyParser";
+
+ /**
+ * Determines the set of MediaType which have been configured with an
EmptyParser.
+ *
+ * @param configStream stream for tika config
+ * @return set of MediaTypes which are not indexed
+ */
+ public static Set<MediaType> getNonIndexedMediaTypes(InputStream
configStream) throws
+ TikaException, IOException, SAXException {
+ Set<MediaType> result = new HashSet<>();
+ Element element =
getBuilder().parse(configStream).getDocumentElement();
+ NodeList nodes = element.getElementsByTagName("parsers");
+ if (nodes.getLength() == 1) {
+ Node parentNode = nodes.item(0);
+ NodeList parsersNodes = parentNode.getChildNodes();
+ for (int i = 0; i < parsersNodes.getLength(); i++) {
+ Node node = parsersNodes.item(i);
+ if (node instanceof Element) {
+ String className = ((Element) node).getAttribute("class");
+ if (EMPTY_PARSER.equals(className)) {
+ NodeList mimes = ((Element)
node).getElementsByTagName("mime");
+ parseMimeTypes(result, mimes);
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+
+ private static void parseMimeTypes(Set<MediaType> result, NodeList mimes) {
+ /*
+ <parser class="org.apache.tika.parser.EmptyParser">
+ <mime>application/x-archive</mime>
+ <mime>application/x-bzip</mime>
+ <mime>application/x-bzip2</mime>
+ </parser>
+ */
+ for (int j = 0; j < mimes.getLength(); j++) {
+ Node mime = mimes.item(j);
+ if (mime instanceof Element) {
+ String mimeValue = mime.getTextContent();
+ mimeValue = Strings.emptyToNull(mimeValue);
+ if (mimeValue != null) {
+ MediaType mediaType = MediaType.parse(mimeValue.trim());
+ if (mediaType != null) {
+ result.add(mediaType);
+ }
+ }
+ }
+ }
+ }
+
+ private static DocumentBuilder getBuilder() throws TikaException {
+ return new ParseContext().getDocumentBuilder();
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/TikaParserConfig.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/IndexHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/IndexHelper.java?rev=1831979&r1=1831978&r2=1831979&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/IndexHelper.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/util/IndexHelper.java
Mon May 21 17:06:10 2018
@@ -151,7 +151,7 @@ public class IndexHelper {
@Nonnull NodeBuilder index, @Nonnull String name, String type,
@Nonnull Set<String> includes,
@Nonnull String async) {
- checkArgument(!includes.isEmpty(), "Lucene property index " +
+ checkArgument(!includes.isEmpty(), "Fulltext property index " +
"requires explicit list of property names to be indexed");
index = index.child(name);
Added:
jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/jackrabbit/oak/plugins/index/search/tika-config.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/jackrabbit/oak/plugins/index/search/tika-config.xml?rev=1831979&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/jackrabbit/oak/plugins/index/search/tika-config.xml
(added)
+++
jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/jackrabbit/oak/plugins/index/search/tika-config.xml
Mon May 21 17:06:10 2018
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<properties>
+ <detectors>
+ <detector class="org.apache.tika.detect.TypeDetector"/>
+ </detectors>
+ <parsers>
+ <parser class="org.apache.tika.parser.DefaultParser"/>
+ <parser class="org.apache.tika.parser.EmptyParser">
+ <!-- Disable package extraction as it's too resource-intensive -->
+ <mime>application/x-archive</mime>
+ <mime>application/x-bzip</mime>
+ <mime>application/x-bzip2</mime>
+ <mime>application/x-cpio</mime>
+ <mime>application/x-gtar</mime>
+ <mime>application/x-gzip</mime>
+ <mime>application/x-tar</mime>
+ <mime>application/zip</mime>
+ <!-- Disable image extraction as there's no text to be found -->
+ <mime>image/bmp</mime>
+ <mime>image/gif</mime>
+ <mime>image/jpeg</mime>
+ <mime>image/png</mime>
+ <mime>image/tiff</mime>
+ <mime>image/vnd.wap.wbmp</mime>
+ <mime>image/x-icon</mime>
+ <mime>image/x-psd</mime>
+ <mime>image/x-xcf</mime>
+ </parser>
+ </parsers>
+ <service-loader initializableProblemHandler="ignore" dynamic="true"/>
+</properties>
Propchange:
jackrabbit/oak/trunk/oak-search/src/main/resources/org/apache/jackrabbit/oak/plugins/index/search/tika-config.xml
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/TestUtil.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/TestUtil.java?rev=1831979&r1=1831978&r2=1831979&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/TestUtil.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/TestUtil.java
Mon May 21 17:06:10 2018
@@ -248,7 +248,6 @@ public class TestUtil {
private static PropertyState createAsyncProperty(IndexingMode
indexingMode) {
switch(indexingMode) {
- case NRT :
case SYNC :
return createAsyncProperty(indexingMode.asyncValueName());
case ASYNC:
