Author: chetanm
Date: Thu Sep 15 07:20:16 2016
New Revision: 1760865

URL: http://svn.apache.org/viewvc?rev=1760865&view=rev
Log:
OAK-4412 - Lucene hybrid index

Enforce a limit on max number of indexable documents held in memory to ensure 
that memory consumption is bounded for large commits. For now its configurable 
via system property only

Modified:
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactoryTest.java

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java?rev=1760865&r1=1760864&r2=1760865&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
 Thu Sep 15 07:20:16 2016
@@ -57,6 +57,13 @@ public class LuceneIndexEditorProvider i
     private final LuceneIndexWriterFactory indexWriterFactory;
     private final IndexTracker indexTracker;
 
+    /**
+     * Number of indexed Lucene document that can be held in memory
+     * This ensures that for very large commit memory consumption
+     * is bounded
+     */
+    private int inMemoryDocsLimit = 
Integer.getInteger("oak.lucene.inMemoryDocsLimit", 500);
+
     public LuceneIndexEditorProvider() {
         this(null);
     }
@@ -121,7 +128,7 @@ public class LuceneIndexEditorProvider i
 
                 //TODO Also check if index has been done once
 
-                writerFactory = new LocalIndexWriterFactory(indexingContext);
+                writerFactory = new LocalIndexWriterFactory(indexingContext, 
inMemoryDocsLimit);
 
                 //IndexDefinition from tracker might differ from one passed 
here for reindexing
                 //case which should be fine. However reusing existing 
definition would avoid
@@ -154,4 +161,8 @@ public class LuceneIndexEditorProvider i
     ExtractedTextCache getExtractedTextCache() {
         return extractedTextCache;
     }
+
+    public void setInMemoryDocsLimit(int inMemoryDocsLimit) {
+        this.inMemoryDocsLimit = inMemoryDocsLimit;
+    }
 }

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java?rev=1760865&r1=1760864&r2=1760865&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java
 Thu Sep 15 07:20:16 2016
@@ -35,10 +35,12 @@ public class LocalIndexWriterFactory imp
     public static final String COMMIT_PROCESSED_BY_LOCAL_LUCENE_EDITOR = 
"commitProcessedByLocalLuceneEditor";
     private final IndexingContext indexingContext;
     private final CommitContext commitContext;
+    private final int inMemoryDocsLimit;
 
-    public LocalIndexWriterFactory(IndexingContext indexingContext) {
+    public LocalIndexWriterFactory(IndexingContext indexingContext, int 
inMemoryDocsLimit) {
         this.indexingContext = indexingContext;
         this.commitContext = getCommitContext(indexingContext);
+        this.inMemoryDocsLimit = inMemoryDocsLimit;
     }
 
     private LuceneDocumentHolder getDocumentHolder(){
@@ -100,8 +102,12 @@ public class LocalIndexWriterFactory imp
                             "mode apart from 'sync' and 'nrt'");
                 }
             }
-            //TODO [hybrid] checks about the size. If too many drop
-            //However for truly sync case hold on
+
+            if (definition.isNRTIndexingEnabled()
+                    && 
getDocumentHolder().checkLimitAndLogWarning(inMemoryDocsLimit)){
+               return;
+            }
+
             docList.add(luceneDoc);
         }
     }

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java?rev=1760865&r1=1760864&r2=1760865&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java
 Thu Sep 15 07:20:16 2016
@@ -25,12 +25,16 @@ import java.util.Map;
 
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ListMultimap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 class LuceneDocumentHolder {
+    private static final Logger log = 
LoggerFactory.getLogger(LuceneDocumentHolder.class);
     public static final String NAME = "oak.lucene.documentHolder";
 
     private final ListMultimap<String, LuceneDoc> nrtIndexedList = 
ArrayListMultimap.create();
     private final ListMultimap<String, LuceneDoc> syncIndexedList = 
ArrayListMultimap.create();
+    private boolean limitWarningLogged;
 
     public List<LuceneDoc> getNRTIndexedDocList(String indexPath) {
         return nrtIndexedList.get(indexPath);
@@ -47,4 +51,16 @@ class LuceneDocumentHolder {
     public Map<String, Collection<LuceneDoc>> getSyncIndexedDocs(){
         return syncIndexedList.asMap();
     }
+
+    public boolean checkLimitAndLogWarning(int maxSize){
+        if (nrtIndexedList.size() >= maxSize){
+            if (!limitWarningLogged){
+                log.warn("Number of in memory documents meant for hybrid 
indexing has " +
+                        "exceeded limit [{}]. Some documents would be 
dropped", maxSize);
+                limitWarningLogged = true;
+            }
+            return true;
+        }
+        return false;
+    }
 }

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactoryTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactoryTest.java?rev=1760865&r1=1760864&r2=1760865&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactoryTest.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactoryTest.java
 Thu Sep 15 07:20:16 2016
@@ -52,10 +52,11 @@ public class LocalIndexWriterFactoryTest
     private EditorHook syncHook;
     private EditorHook asyncHook;
     private CommitInfo info;
+    private LuceneIndexEditorProvider editorProvider;
 
     @Before
     public void setUp() throws IOException {
-        IndexEditorProvider editorProvider = new LuceneIndexEditorProvider(
+        editorProvider = new LuceneIndexEditorProvider(
                 null,
                 null,
                 null,
@@ -153,6 +154,21 @@ public class LocalIndexWriterFactoryTest
         assertEquals(0, 
holder.getNRTIndexedDocList("/oak:index/fooIndex").size());
     }
 
+    @Test
+    public void inMemoryDocLimit() throws Exception{
+        NodeState indexed = createAndPopulateAsyncIndex(IndexingMode.NRT);
+        editorProvider.setInMemoryDocsLimit(5);
+        builder = indexed.builder();
+        for (int i = 0; i < 10; i++) {
+            builder.child("b" + i).setProperty("foo", "bar");
+        }
+        NodeState after = builder.getNodeState();
+        syncHook.processCommit(indexed, after, newCommitInfo());
+
+        LuceneDocumentHolder holder = getHolder();
+        assertEquals(5, 
holder.getNRTIndexedDocList("/oak:index/fooIndex").size());
+    }
+
     private NodeState createAndPopulateAsyncIndex(IndexingMode indexingMode) 
throws CommitFailedException {
         createIndexDefinition("fooIndex", indexingMode);
 


Reply via email to