Author: chetanm
Date: Thu Sep 15 07:19:34 2016
New Revision: 1760861

URL: http://svn.apache.org/viewvc?rev=1760861&view=rev
Log:
OAK-4412 - Lucene hybrid index

Adding support for synchronous lucene property hybrid index
-- Introduced new enum for IndexingMode - SYNC, NRT, ASYNC
-- If the index definition is enabled for 'sync' then lucene documents
     would be added to index and readers refreshed after write immediately
-- Moved the stats collection to Queue

Modified:
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactory.java
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/TestUtil.java
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueueTest.java
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/HybridIndexTest.java
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactoryTest.java
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
 Thu Sep 15 07:19:34 2016
@@ -231,6 +231,7 @@ public final class IndexDefinition imple
     private final String indexPath;
 
     private final boolean nrtIndexMode;
+    private final boolean syncIndexMode;
 
     @Nullable
     private final String uid;
@@ -305,6 +306,7 @@ public final class IndexDefinition imple
         this.suggestEnabled = evaluateSuggestionEnabled();
         this.spellcheckEnabled = evaluateSpellcheckEnabled();
         this.nrtIndexMode = supportsNRTIndexing(defn);
+        this.syncIndexMode = supportsSyncIndexing(defn);
     }
 
     public NodeState getDefinitionNodeState() {
@@ -440,6 +442,11 @@ public final class IndexDefinition imple
         return nrtIndexMode;
     }
 
+    public boolean isSyncIndexingEnabled() {
+        return syncIndexMode;
+    }
+
+
     @Override
     public String toString() {
         return "Lucene Index : " + indexName;
@@ -1576,6 +1583,10 @@ public final class IndexDefinition imple
         return supportsIndexingMode(new ReadOnlyBuilder(defn), "nrt");
     }
 
+    private static boolean supportsSyncIndexing(NodeState defn) {
+        return supportsIndexingMode(new ReadOnlyBuilder(defn), "sync");
+    }
+
     public static boolean supportsSyncOrNRTIndexing(NodeBuilder defn) {
        return supportsIndexingMode(defn, "nrt") || supportsIndexingMode(defn, 
"sync");
     }

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
 Thu Sep 15 07:19:34 2016
@@ -148,11 +148,17 @@ public class IndexNode {
 
     public void refreshReadersIfRequired() {
         if (refreshPolicy.shouldRefresh()){
-            indexSearcher = new IndexSearcher(createReader());
-            log.debug("Refreshed reader for index [{}]", definition);
+            refreshReaders();
         }
     }
 
+    public void refreshReaders(){
+        //TODO Instead of refresh on write switch on refresh on read
+        //for sync indexes
+        indexSearcher = new IndexSearcher(createReader());
+        log.debug("Refreshed reader for index [{}]", definition);
+    }
+
     private LuceneIndexReader getDefaultReader(){
         //TODO This is still required to support Suggester, Spellcheck etc 
OAK-4643
         return readers.get(0);

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
 Thu Sep 15 07:19:34 2016
@@ -16,14 +16,26 @@
  */
 package org.apache.jackrabbit.oak.plugins.index.lucene;
 
-import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
 import org.apache.lucene.util.Version;
 
 public interface LuceneIndexConstants {
 
+    enum IndexingMode {
+        SYNC,
+        NRT,
+        ASYNC;
+
+        public String asyncValueName(){
+            return name().toLowerCase();
+        }
+
+        public static IndexingMode from(String indexingMode){
+            return valueOf(indexingMode.toUpperCase());
+        }
+    }
+
     String TYPE_LUCENE = "lucene";
 
     String INDEX_DATA_CHILD_NAME = ":data";

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
 Thu Sep 15 07:19:34 2016
@@ -40,6 +40,7 @@ import org.apache.jackrabbit.oak.plugins
 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
 import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
 import org.apache.jackrabbit.oak.stats.CounterStats;
+import org.apache.jackrabbit.oak.stats.MeterStats;
 import org.apache.jackrabbit.oak.stats.StatisticsProvider;
 import org.apache.jackrabbit.oak.stats.StatsOptions;
 import org.apache.lucene.index.IndexableField;
@@ -55,6 +56,8 @@ public class DocumentQueue implements Cl
     private final BlockingQueue<LuceneDoc> docsQueue;
     private final Executor executor;
     private final CounterStats queueSizeStats;
+    private final MeterStats added;
+    private final MeterStats dropped;
 
     /**
      * Time in millis for which add call to queue
@@ -108,11 +111,8 @@ public class DocumentQueue implements Cl
                         docsPerIndex.get(doc.indexPath).add(doc);
                     }
 
-                    //If required it can optimized by indexing diff indexes in 
parallel
-                    //Something to consider if it becomes a bottleneck
-                    for (Map.Entry<String, Collection<LuceneDoc>> e : 
docsPerIndex.asMap().entrySet()) {
-                        processDocs(e.getKey(), e.getValue());
-                    }
+                    addAllSynchronously(docsPerIndex.asMap(), false);
+
                     currentTask.onComplete(completionHandler);
                 } catch (Throwable t) {
                     exceptionHandler.uncaughtException(Thread.currentThread(), 
t);
@@ -138,6 +138,8 @@ public class DocumentQueue implements Cl
         this.executor = executor;
         this.offerTimeMillis = 100; //Wait for at most 100 mills while adding 
stuff to queue
         this.queueSizeStats = sp.getCounterStats("HYBRID_QUEUE_SIZE", 
StatsOptions.DEFAULT);
+        this.added = sp.getMeter("HYBRID_ADDED", StatsOptions.DEFAULT);
+        this.dropped = sp.getMeter("HYBRID_DROPPED", StatsOptions.DEFAULT);
     }
 
     public boolean add(LuceneDoc doc){
@@ -154,17 +156,28 @@ public class DocumentQueue implements Cl
         currentTask.onComplete(completionHandler);
         if (added) {
             queueSizeStats.inc();
+        } else {
+            dropped.mark();
         }
         return added;
     }
 
+    public void addAllSynchronously(Map<String, Collection<LuceneDoc>> 
docsPerIndex, boolean alwaysRefreshReaders) {
+        //If required it can optimized by indexing diff indexes in parallel
+        //Something to consider if it becomes a bottleneck
+        for (Map.Entry<String, Collection<LuceneDoc>> e : 
docsPerIndex.entrySet()) {
+            processDocs(e.getKey(), e.getValue(), alwaysRefreshReaders);
+            added.mark(e.getValue().size());
+        }
+    }
+
     List<LuceneDoc> getQueuedDocs(){
         List<LuceneDoc> docs = Lists.newArrayList();
         docs.addAll(docsQueue);
         return docs;
     }
 
-    private void processDocs(String indexPath, Iterable<LuceneDoc> docs){
+    private void processDocs(String indexPath, Iterable<LuceneDoc> docs, 
boolean alwaysRefreshReaders){
 
         //Drop the write call if stopped
         if (stopped) {
@@ -194,7 +207,11 @@ public class DocumentQueue implements Cl
                 }
                 log.trace("Updated index with doc {}", doc);
             }
-            indexNode.refreshReadersIfRequired();
+            if (alwaysRefreshReaders) {
+                indexNode.refreshReaders();
+            } else {
+                indexNode.refreshReadersIfRequired();
+            }
         } catch (Exception e) {
             //For now we just log it. Later we need to see if frequent error 
then to
             //temporarily disable indexing for this index

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java
 Thu Sep 15 07:19:34 2016
@@ -26,22 +26,16 @@ import org.apache.jackrabbit.oak.spi.com
 import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
 import org.apache.jackrabbit.oak.spi.commit.Observer;
 import org.apache.jackrabbit.oak.spi.state.NodeState;
-import org.apache.jackrabbit.oak.stats.MeterStats;
 import org.apache.jackrabbit.oak.stats.StatisticsProvider;
-import org.apache.jackrabbit.oak.stats.StatsOptions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class LocalIndexObserver implements Observer{
     private final Logger log = LoggerFactory.getLogger(getClass());
     private final DocumentQueue docQueue;
-    private final MeterStats added;
-    private final MeterStats dropped;
 
     public LocalIndexObserver(DocumentQueue docQueue, StatisticsProvider sp) {
         this.docQueue = docQueue;
-        this.added = sp.getMeter("HYBRID_ADDED", StatsOptions.DEFAULT);
-        this.dropped = sp.getMeter("HYBRID_DROPPED", StatsOptions.DEFAULT);
     }
 
     @Override
@@ -63,20 +57,23 @@ public class LocalIndexObserver implemen
             return;
         }
 
-        int addedCount = 0, droppedCount = 0;
+        int droppedCount = 0;
         for (LuceneDoc doc : holder.getNRTIndexedDocs()){
-            if (docQueue.add(doc)) {
-                addedCount++;
-            } else {
+            if (!docQueue.add(doc)) {
                 droppedCount++;
             }
         }
 
-        added.mark(addedCount);
-        dropped.mark(droppedCount);
+        //After nrt docs add all sync indexed docs
+        //Doing it *after* ensures thar nrt index might catch
+        //up by the time sync one are finished
+        docQueue.addAllSynchronously(holder.getSyncIndexedDocs(), true);
+
         if (droppedCount > 0){
             //TODO Ensure that log do not flood
             log.warn("Dropped [{}] docs from indexing as queue is full", 
droppedCount);
         }
+
+        //TODO Remove the holder once processing is done
     }
 }

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java
 Thu Sep 15 07:19:34 2016
@@ -91,7 +91,14 @@ public class LocalIndexWriterFactory imp
 
         private void addLuceneDoc(LuceneDoc luceneDoc) {
             if (docList == null){
-                docList = 
getDocumentHolder().getNRTIndexedDocList(indexingContext.getIndexPath());
+                if (definition.isSyncIndexingEnabled()){
+                    docList = 
getDocumentHolder().getSyncIndexedDocList(indexingContext.getIndexPath());
+                } else if (definition.isNRTIndexingEnabled()){
+                    docList = 
getDocumentHolder().getNRTIndexedDocList(indexingContext.getIndexPath());
+                } else {
+                    throw new IllegalStateException("Should not be invoked for 
any other indexing " +
+                            "mode apart from 'sync' and 'nrt'");
+                }
             }
             //TODO [hybrid] checks about the size. If too many drop
             //However for truly sync case hold on

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java
 Thu Sep 15 07:19:34 2016
@@ -19,7 +19,9 @@
 
 package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid;
 
+import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ListMultimap;
@@ -28,6 +30,7 @@ class LuceneDocumentHolder {
     public static final String NAME = "oak.lucene.documentHolder";
 
     private final ListMultimap<String, LuceneDoc> nrtIndexedList = 
ArrayListMultimap.create();
+    private final ListMultimap<String, LuceneDoc> syncIndexedList = 
ArrayListMultimap.create();
 
     public List<LuceneDoc> getNRTIndexedDocList(String indexPath) {
         return nrtIndexedList.get(indexPath);
@@ -36,4 +39,12 @@ class LuceneDocumentHolder {
     public Iterable<LuceneDoc> getNRTIndexedDocs(){
         return nrtIndexedList.values();
     }
+
+    public List<LuceneDoc> getSyncIndexedDocList(String indexPath) {
+        return syncIndexedList.get(indexPath);
+    }
+
+    public Map<String, Collection<LuceneDoc>> getSyncIndexedDocs(){
+        return syncIndexedList.asMap();
+    }
 }

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactory.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactory.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactory.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactory.java
 Thu Sep 15 07:19:34 2016
@@ -58,7 +58,7 @@ public class NRTIndexFactory implements
     // but still mark it synchronized for safety
     @CheckForNull
     public synchronized NRTIndex createIndex(IndexDefinition definition) {
-        if (!definition.isNRTIndexingEnabled()){
+        if (!(definition.isNRTIndexingEnabled() || 
definition.isSyncIndexingEnabled())){
             return null;
         }
         String indexPath = definition.getIndexPathFromConfig();

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
 Thu Sep 15 07:19:34 2016
@@ -874,6 +874,14 @@ public class IndexDefinitionTest {
         assertTrue(idxDefn.isNRTIndexingEnabled());
     }
 
+    @Test
+    public void sync() throws Exception{
+        TestUtil.enableIndexingMode(builder, 
LuceneIndexConstants.IndexingMode.SYNC);
+        IndexDefinition idxDefn = new IndexDefinition(root, 
builder.getNodeState());
+        assertFalse(idxDefn.isNRTIndexingEnabled());
+        assertTrue(idxDefn.isSyncIndexingEnabled());
+    }
+
     //TODO indexesAllNodesOfMatchingType - with nullCheckEnabled
 
     private static IndexingRule getRule(IndexDefinition defn, String typeName){

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/TestUtil.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/TestUtil.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/TestUtil.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/TestUtil.java
 Thu Sep 15 07:19:34 2016
@@ -33,11 +33,13 @@ import org.apache.jackrabbit.JcrConstant
 import org.apache.jackrabbit.api.JackrabbitRepository;
 import org.apache.jackrabbit.oak.api.Blob;
 import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.jackrabbit.oak.api.Root;
 import org.apache.jackrabbit.oak.api.Tree;
 import org.apache.jackrabbit.oak.api.Type;
 import org.apache.jackrabbit.oak.commons.PathUtils;
 import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import 
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.IndexingMode;
 import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper;
 import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
 import org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState;
@@ -222,12 +224,38 @@ public class TestUtil {
     }
 
     public static NodeBuilder enableNRTIndexing(NodeBuilder builder){
-        builder.setProperty(createProperty(IndexConstants.ASYNC_PROPERTY_NAME, 
of("nrt" , "async"), STRINGS));
+        builder.setProperty(createAsyncProperty("nrt"));
         return builder;
     }
 
     public static Tree enableNRTIndexing(Tree tree){
-        tree.setProperty(createProperty(IndexConstants.ASYNC_PROPERTY_NAME, 
of("nrt" , "async"), STRINGS));
+        tree.setProperty(createAsyncProperty("nrt"));
         return tree;
     }
+
+    public static NodeBuilder enableIndexingMode(NodeBuilder builder, 
IndexingMode indexingMode){
+        builder.setProperty(createAsyncProperty(indexingMode));
+        return builder;
+    }
+
+    public static Tree enableIndexingMode(Tree tree, IndexingMode 
indexingMode){
+        tree.setProperty(createAsyncProperty(indexingMode));
+        return tree;
+    }
+
+    private static PropertyState createAsyncProperty(String indexingMode) {
+        return createProperty(IndexConstants.ASYNC_PROPERTY_NAME, 
of(indexingMode , "async"), STRINGS);
+    }
+
+    private static PropertyState createAsyncProperty(IndexingMode 
indexingMode) {
+        switch(indexingMode) {
+            case NRT  :
+            case SYNC :
+                return createAsyncProperty(indexingMode.asyncValueName());
+            case ASYNC:
+                return createProperty(IndexConstants.ASYNC_PROPERTY_NAME, 
of("async"), STRINGS);
+            default:
+                throw new IllegalArgumentException("Unknown mode " + 
indexingMode);
+        }
+    }
 }

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueueTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueueTest.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueueTest.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueueTest.java
 Thu Sep 15 07:19:34 2016
@@ -27,8 +27,10 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Stopwatch;
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ListMultimap;
 import org.apache.jackrabbit.oak.api.CommitFailedException;
 import org.apache.jackrabbit.oak.core.SimpleCommitContext;
 import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
@@ -214,6 +216,33 @@ public class DocumentQueueTest {
         assertEquals(4, td.totalHits);
     }
 
+    @Test
+    public void addAllSync() throws Exception{
+        ListMultimap<String, LuceneDoc> docs = ArrayListMultimap.create();
+        tracker = createTracker();
+        NodeState indexed = createAndPopulateAsyncIndex();
+        tracker.update(indexed);
+
+        DocumentQueue queue = new DocumentQueue(2, tracker, 
sameThreadExecutor());
+
+        TopDocs td = doSearch("bar");
+        assertEquals(1, td.totalHits);
+
+        docs.get("/oak:index/fooIndex").add(createDoc("/a/c", "bar"));
+        queue.addAllSynchronously(docs.asMap(), true);
+
+        td = doSearch("bar");
+        assertEquals(2, td.totalHits);
+
+        docs.clear();
+
+        docs.get("/oak:index/fooIndex").add(createDoc("/a/d", "bar"));
+        queue.addAllSynchronously(docs.asMap(), true);
+
+        td = doSearch("bar");
+        assertEquals(3, td.totalHits);
+    }
+
     //@Test
     public void benchMarkIndexWriter() throws Exception{
         Executor executor = Executors.newFixedThreadPool(5);
@@ -228,10 +257,15 @@ public class DocumentQueueTest {
 
         DocumentQueue queue = new DocumentQueue(1000, tracker, executor);
 
-        Document d1 = new Document();
-        d1.add(newPathField("/a/b"));
-        d1.add(new StringField("foo", "a", Field.Store.NO));
-        LuceneDoc doc = LuceneDoc.forUpdate("/oak:index/fooIndex", "/a/b", d1);
+        /*
+            Sample output
+            [nrt] Time taken for 10000 is 639.3 ms with waits 1
+            [sync] Time taken for 10000 is 30.34 s
+
+            Refreshing reader after every commit would slow down things
+         */
+
+        LuceneDoc doc = createDoc("/a/b", "a");
         int numDocs = 10000;
         Stopwatch w = Stopwatch.createStarted();
         int waitCount = 0;
@@ -241,7 +275,16 @@ public class DocumentQueueTest {
             }
         }
 
-        System.out.printf("%nTime taken for %d is %s with waits %d", numDocs, 
w, waitCount);
+        System.out.printf("%n[nrt] Time taken for %d is %s with waits %d%n", 
numDocs, w, waitCount);
+
+        w = Stopwatch.createStarted();
+        for (int i = 0; i < numDocs; i++) {
+            ListMultimap<String, LuceneDoc> docs = ArrayListMultimap.create();
+            docs.get("/oak:index/fooIndex").add(doc);
+            queue.addAllSynchronously(docs.asMap(), true);
+        }
+        System.out.printf("%n[sync] Time taken for %d is %s%n", numDocs, w);
+
     }
 
     private NodeState doAsyncIndex(NodeState current, String childName, String 
fooValue) throws CommitFailedException {
@@ -262,10 +305,15 @@ public class DocumentQueueTest {
     }
 
     private void addDoc(DocumentQueue queue, String docPath, String fooValue) {
+        LuceneDoc doc = createDoc(docPath, fooValue);
+        queue.add(doc);
+    }
+
+    private static LuceneDoc createDoc(String docPath, String fooValue) {
         Document d1 = new Document();
         d1.add(newPathField(docPath));
         d1.add(new StringField("foo", fooValue, Field.Store.NO));
-        queue.add(LuceneDoc.forUpdate("/oak:index/fooIndex", docPath, d1));
+        return LuceneDoc.forUpdate("/oak:index/fooIndex", docPath, d1);
     }
 
     private IndexTracker createTracker() throws IOException {
@@ -295,6 +343,8 @@ public class DocumentQueueTest {
     private void createIndexDefinition(String idxName) {
         NodeBuilder idx = 
newLucenePropertyIndexDefinition(builder.child("oak:index"),
                 idxName, ImmutableSet.of("foo"), "async");
+        //Disable compression
+        //idx.setProperty("codec", "oakCodec");
         TestUtil.enableNRTIndexing(idx);
     }
 

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/HybridIndexTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/HybridIndexTest.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/HybridIndexTest.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/HybridIndexTest.java
 Thu Sep 15 07:19:34 2016
@@ -39,6 +39,7 @@ import org.apache.jackrabbit.oak.plugins
 import 
org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider;
 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import 
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.IndexingMode;
 import 
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
 import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil;
@@ -186,6 +187,29 @@ public class HybridIndexTest extends Abs
 
     }
 
+    @Test
+    public void hybridIndexSync() throws Exception{
+        String idxName = "hybridtest";
+        Tree idx = createIndex(root.getTree("/"), idxName, 
Collections.singleton("foo"));
+        TestUtil.enableIndexingMode(idx, IndexingMode.SYNC);
+        root.commit();
+
+        //Get initial indexing done as local indexing only work
+        //for incremental indexing
+        createPath("/a").setProperty("foo", "bar");
+        root.commit();
+
+        runAsyncIndex();
+
+        setTraversalEnabled(false);
+        assertQuery("select [jcr:path] from [nt:base] where [foo] = 'bar'", 
of("/a"));
+
+        //Add new node. This should get immediately reelected as its a sync 
index
+        createPath("/b").setProperty("foo", "bar");
+        root.commit();
+        assertQuery("select [jcr:path] from [nt:base] where [foo] = 'bar'", 
of("/a", "/b"));
+    }
+
     private void runAsyncIndex() {
         Runnable async = WhiteboardUtils.getService(wb, Runnable.class, new 
Predicate<Runnable>() {
             @Override

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactoryTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactoryTest.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactoryTest.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactoryTest.java
 Thu Sep 15 07:19:34 2016
@@ -25,9 +25,9 @@ import com.google.common.collect.Immutab
 import com.google.common.collect.ImmutableSet;
 import org.apache.jackrabbit.oak.api.CommitFailedException;
 import org.apache.jackrabbit.oak.core.SimpleCommitContext;
-import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
 import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
 import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider;
+import 
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.IndexingMode;
 import 
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
 import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil;
 import org.apache.jackrabbit.oak.spi.commit.CommitContext;
@@ -40,11 +40,8 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import static com.google.common.collect.ImmutableSet.of;
-import static org.apache.jackrabbit.oak.api.Type.STRINGS;
 import static 
org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper.newLucenePropertyIndexDefinition;
 import static 
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
-import static 
org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty;
 import static 
org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent.INITIAL_CONTENT;
 import static org.junit.Assert.*;
 
@@ -75,7 +72,7 @@ public class LocalIndexWriterFactoryTest
 
     @Test
     public void ignoreReindexCase() throws Exception{
-        createIndexDefinition("fooIndex");
+        createIndexDefinition("fooIndex", IndexingMode.NRT);
 
         builder.child("a").setProperty("foo", "bar");
         NodeState after = builder.getNodeState();
@@ -89,7 +86,7 @@ public class LocalIndexWriterFactoryTest
 
     @Test
     public void holderNotInitializedUnlessIndexed() throws Exception{
-        NodeState indexed = createAndPopulateAsyncIndex();
+        NodeState indexed = createAndPopulateAsyncIndex(IndexingMode.NRT);
         builder = indexed.builder();
         builder.child("b");
         NodeState after = builder.getNodeState();
@@ -103,7 +100,7 @@ public class LocalIndexWriterFactoryTest
 
     @Test
     public void localIndexWriter() throws Exception{
-        NodeState indexed = createAndPopulateAsyncIndex();
+        NodeState indexed = createAndPopulateAsyncIndex(IndexingMode.NRT);
         builder = indexed.builder();
         builder.child("b").setProperty("foo", "bar");
         builder.child("c").setProperty("foo", "bar");
@@ -120,7 +117,7 @@ public class LocalIndexWriterFactoryTest
 
     @Test
     public void mutlipleIndex() throws Exception{
-        NodeState indexed = createAndPopulateTwoAsyncIndex();
+        NodeState indexed = createAndPopulateTwoAsyncIndex(IndexingMode.NRT);
         builder = indexed.builder();
         builder.child("b").setProperty("foo", "bar");
         builder.child("c").setProperty("bar", "foo");
@@ -139,8 +136,25 @@ public class LocalIndexWriterFactoryTest
 
     }
 
-    private NodeState createAndPopulateAsyncIndex() throws 
CommitFailedException {
-        createIndexDefinition("fooIndex");
+    @Test
+    public void syncIndexing() throws Exception{
+        NodeState indexed = createAndPopulateAsyncIndex(IndexingMode.SYNC);
+        builder = indexed.builder();
+        builder.child("b").setProperty("foo", "bar");
+        builder.child("c").setProperty("foo", "bar");
+        NodeState after = builder.getNodeState();
+        syncHook.processCommit(indexed, after, newCommitInfo());
+
+        LuceneDocumentHolder holder = getHolder();
+        assertNotNull(holder);
+
+        //2 add none for delete
+        assertEquals(2, 
holder.getSyncIndexedDocList("/oak:index/fooIndex").size());
+        assertEquals(0, 
holder.getNRTIndexedDocList("/oak:index/fooIndex").size());
+    }
+
+    private NodeState createAndPopulateAsyncIndex(IndexingMode indexingMode) 
throws CommitFailedException {
+        createIndexDefinition("fooIndex", indexingMode);
 
         //Have some stuff to be indexed
         builder.child("a").setProperty("foo", "bar");
@@ -148,9 +162,9 @@ public class LocalIndexWriterFactoryTest
         return asyncHook.processCommit(EMPTY_NODE, after, newCommitInfo());
     }
 
-    private NodeState createAndPopulateTwoAsyncIndex() throws 
CommitFailedException {
-        createIndexDefinition("fooIndex");
-        createIndexDefinition("barIndex");
+    private NodeState createAndPopulateTwoAsyncIndex(IndexingMode 
indexingMode) throws CommitFailedException {
+        createIndexDefinition("fooIndex", indexingMode);
+        createIndexDefinition("barIndex", indexingMode);
 
         //Have some stuff to be indexed
         builder.child("a").setProperty("foo", "bar");
@@ -174,10 +188,10 @@ public class LocalIndexWriterFactoryTest
         return info;
     }
 
-    private void createIndexDefinition(String idxName) {
+    private void createIndexDefinition(String idxName, IndexingMode 
indexingMode) {
         NodeBuilder idx = 
newLucenePropertyIndexDefinition(builder.child("oak:index"),
                 idxName, ImmutableSet.of("foo"), "async");
-        TestUtil.enableNRTIndexing(idx);
+        TestUtil.enableIndexingMode(idx, indexingMode);
     }
 
 }
\ No newline at end of file

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java?rev=1760861&r1=1760860&r2=1760861&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java
 Thu Sep 15 07:19:34 2016
@@ -25,6 +25,7 @@ import java.io.IOException;
 import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
+import 
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.IndexingMode;
 import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil;
 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
 import org.apache.jackrabbit.oak.spi.state.NodeState;
@@ -64,8 +65,17 @@ public class NRTIndexFactoryTest {
     }
 
     @Test
-    public void indexCreation() throws Exception{
-        IndexDefinition idxDefn = getSyncIndexDefinition("/foo");
+    public void indexCreationNRT() throws Exception{
+        IndexDefinition idxDefn = getIndexDefinition("/foo", 
IndexingMode.SYNC);
+
+        NRTIndex idx1 = indexFactory.createIndex(idxDefn);
+        assertNotNull(idx1);
+        assertEquals(1, indexFactory.getIndexes("/foo").size());
+    }
+
+    @Test
+    public void indexCreationSync() throws Exception{
+        IndexDefinition idxDefn = getNRTIndexDefinition("/foo");
 
         NRTIndex idx1 = indexFactory.createIndex(idxDefn);
         assertNotNull(idx1);
@@ -74,7 +84,7 @@ public class NRTIndexFactoryTest {
 
     @Test
     public void indexCreationAndCloser() throws Exception{
-        IndexDefinition idxDefn = getSyncIndexDefinition("/foo");
+        IndexDefinition idxDefn = getNRTIndexDefinition("/foo");
 
         NRTIndex idx1 = indexFactory.createIndex(idxDefn);
         assertNotNull(idx1);
@@ -93,7 +103,7 @@ public class NRTIndexFactoryTest {
 
     @Test
     public void closeIndexOnClose() throws Exception{
-        IndexDefinition idxDefn = getSyncIndexDefinition("/foo");
+        IndexDefinition idxDefn = getNRTIndexDefinition("/foo");
 
         NRTIndex idx1 = indexFactory.createIndex(idxDefn);
         NRTIndex idx2 = indexFactory.createIndex(idxDefn);
@@ -105,9 +115,13 @@ public class NRTIndexFactoryTest {
         assertTrue(idx2.isClosed());
     }
 
-    private IndexDefinition getSyncIndexDefinition(String indexPath) {
+    private IndexDefinition getNRTIndexDefinition(String indexPath) {
+       return getIndexDefinition(indexPath, IndexingMode.NRT);
+    }
+
+    private IndexDefinition getIndexDefinition(String indexPath, IndexingMode 
indexingMode) {
         builder.setProperty(IndexConstants.INDEX_PATH, indexPath);
-        TestUtil.enableNRTIndexing(builder);
+        TestUtil.enableIndexingMode(builder, indexingMode);
 
         return new IndexDefinition(root, builder.getNodeState());
     }



Reply via email to