Author: chetanm
Date: Thu Sep 15 07:16:09 2016
New Revision: 1760840

URL: http://svn.apache.org/viewvc?rev=1760840&view=rev
Log:
OAK-4412 - Lucene hybrid index

-- Enable stats collection around queue state
-- Add some extra logging in IndexNode and other parts
-- Minor change in IndexCopier to check for file existence before finding 
length. This avoids throwing of exception within Lucene. This is just for 
goodness!

Modified:
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/HybridIndexTest.java
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserverTest.java

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java?rev=1760840&r1=1760839&r2=1760840&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
 Thu Sep 15 07:16:09 2016
@@ -1208,10 +1208,15 @@ public class IndexCopier implements Copy
      */
     private static long getFileLength(Directory dir, String fileName){
         try{
-            return dir.fileLength(fileName);
-        } catch (Exception e){
-            return -1;
+            //Check for file presence otherwise internally it results in
+            //an exception to be created
+            if (dir.fileExists(fileName)) {
+                return dir.fileLength(fileName);
+            }
+        } catch (Exception ignore){
+
         }
+        return -1;
     }
 
     //~------------------------------------------< CopyOnReadStatsMBean >

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1760840&r1=1760839&r2=1760840&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
 Thu Sep 15 07:16:09 2016
@@ -331,6 +331,7 @@ public final class IndexDefinition imple
         return blobSize;
     }
 
+    @CheckForNull
     public Codec getCodec() {
         return codec;
     }

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java?rev=1760840&r1=1760839&r2=1760840&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
 Thu Sep 15 07:16:09 2016
@@ -43,6 +43,8 @@ import org.apache.lucene.index.MultiRead
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester;
 import org.apache.lucene.store.Directory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class IndexNode {
 
@@ -58,6 +60,8 @@ public class IndexNode {
         return null;
     }
 
+    private static final Logger log = LoggerFactory.getLogger(IndexNode.class);
+
     private final List<LuceneIndexReader> readers;
 
     private final String name;
@@ -145,6 +149,7 @@ public class IndexNode {
     public void refreshReadersIfRequired() {
         if (refreshPolicy.shouldRefresh()){
             indexSearcher = new IndexSearcher(createReader());
+            log.debug("Refreshed reader for index [{}]", definition);
         }
     }
 

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java?rev=1760840&r1=1760839&r2=1760840&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
 Thu Sep 15 07:16:09 2016
@@ -34,6 +34,9 @@ import org.apache.jackrabbit.oak.commons
 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexNode;
 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
 import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
+import org.apache.jackrabbit.oak.stats.CounterStats;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.apache.jackrabbit.oak.stats.StatsOptions;
 import org.apache.lucene.index.IndexableField;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -46,6 +49,7 @@ public class DocumentQueue implements Cl
     private final IndexTracker tracker;
     private final BlockingQueue<LuceneDoc> docsQueue;
     private final Executor executor;
+    private final CounterStats queueSizeStats;
     private volatile boolean stopped;
 
     /**
@@ -75,6 +79,7 @@ public class DocumentQueue implements Cl
                     LuceneDoc doc = docsQueue.poll();
                     if (doc != null && doc != STOP) {
                         processDoc(doc);
+                        queueSizeStats.dec();
                         currentTask.onComplete(completionHandler);
                     }
                 } catch (Throwable t) {
@@ -92,9 +97,14 @@ public class DocumentQueue implements Cl
     };
 
     public DocumentQueue(int maxQueueSize, IndexTracker tracker, Executor 
executor) {
+        this(maxQueueSize, tracker, executor, StatisticsProvider.NOOP);
+    }
+
+    public DocumentQueue(int maxQueueSize, IndexTracker tracker, Executor 
executor, StatisticsProvider sp) {
         this.docsQueue = new LinkedBlockingDeque<>(maxQueueSize);
         this.tracker = tracker;
         this.executor = executor;
+        this.queueSizeStats = sp.getCounterStats("HYBRID_QUEUE_SIZE", 
StatsOptions.DEFAULT);
     }
 
     public boolean add(LuceneDoc doc){
@@ -104,6 +114,9 @@ public class DocumentQueue implements Cl
         // to onComplete are not a problem here since we always pass the same 
value.
         // Thus there is no question as to which of the handlers will 
effectively run.
         currentTask.onComplete(completionHandler);
+        if (added) {
+            queueSizeStats.inc();
+        }
         //TODO log warning when queue is full
         return added;
     }
@@ -136,6 +149,7 @@ public class DocumentQueue implements Cl
             } else {
                 writer.updateDocument(doc.docPath, doc.doc);
             }
+            log.trace("Updated index with doc {}", doc);
             indexNode.refreshReadersIfRequired();
         } catch (Exception e) {
             //For now we just log it. Later we need to see if frequent error 
then to

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java?rev=1760840&r1=1760839&r2=1760840&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java
 Thu Sep 15 07:16:09 2016
@@ -26,12 +26,19 @@ import org.apache.jackrabbit.oak.spi.com
 import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
 import org.apache.jackrabbit.oak.spi.commit.Observer;
 import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.stats.MeterStats;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.apache.jackrabbit.oak.stats.StatsOptions;
 
 public class LocalIndexObserver implements Observer{
     private final DocumentQueue docQueue;
+    private final MeterStats added;
+    private final MeterStats dropped;
 
-    public LocalIndexObserver(DocumentQueue docQueue) {
+    public LocalIndexObserver(DocumentQueue docQueue, StatisticsProvider sp) {
         this.docQueue = docQueue;
+        this.added = sp.getMeter("HYBRID_ADDED", StatsOptions.DEFAULT);
+        this.dropped = sp.getMeter("HYBRID_DROPPED", StatsOptions.DEFAULT);
     }
 
     @Override
@@ -53,8 +60,16 @@ public class LocalIndexObserver implemen
             return;
         }
 
+        int addedCount = 0, droppedCount = 0;
         for (LuceneDoc doc : holder.getAsyncIndexedDocs()){
-            docQueue.add(doc);
+            if (docQueue.add(doc)) {
+                addedCount++;
+            } else {
+                droppedCount++;
+            }
         }
+
+        added.mark(addedCount);
+        dropped.mark(droppedCount);
     }
 }

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/HybridIndexTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/HybridIndexTest.java?rev=1760840&r1=1760839&r2=1760840&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/HybridIndexTest.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/HybridIndexTest.java
 Thu Sep 15 07:16:09 2016
@@ -60,6 +60,7 @@ import org.apache.jackrabbit.oak.spi.sta
 import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
 import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils;
 import org.apache.jackrabbit.oak.stats.Clock;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -107,7 +108,7 @@ public class HybridIndexTest extends Abs
                 mip);
 
         queue = new DocumentQueue(100, tracker, sameThreadExecutor());
-        LocalIndexObserver localIndexObserver = new LocalIndexObserver(queue);
+        LocalIndexObserver localIndexObserver = new LocalIndexObserver(queue, 
StatisticsProvider.NOOP);
 
         nodeStore = new MemoryNodeStore();
         Oak oak = new Oak(nodeStore)

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserverTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserverTest.java?rev=1760840&r1=1760839&r2=1760840&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserverTest.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserverTest.java
 Thu Sep 15 07:16:09 2016
@@ -26,6 +26,7 @@ import org.apache.jackrabbit.oak.core.Si
 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
 import org.apache.jackrabbit.oak.spi.commit.CommitContext;
 import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -47,7 +48,7 @@ public class LocalIndexObserverTest {
     @Before
     public void setUp(){
         collectingQueue = new DocumentQueue(10, tracker, NOOP_EXECUTOR);
-        observer = new LocalIndexObserver(collectingQueue);
+        observer = new LocalIndexObserver(collectingQueue, 
StatisticsProvider.NOOP);
     }
 
     @Test


Reply via email to