Author: chetanm
Date: Thu Aug 10 08:41:49 2017
New Revision: 1804642

URL: http://svn.apache.org/viewvc?rev=1804642&view=rev
Log:
OAK-6523 - Tune DocumentNodeStore setup for indexing flow for index command

Added:
    
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexDocumentBuilderCustomizer.java
   (with props)
Modified:
    
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java

Modified: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java?rev=1804642&r1=1804641&r2=1804642&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
 Thu Aug 10 08:41:49 2017
@@ -39,16 +39,19 @@ import org.apache.felix.inventory.Format
 import org.apache.jackrabbit.oak.api.CommitFailedException;
 import org.apache.jackrabbit.oak.plugins.index.importer.IndexDefinitionUpdater;
 import org.apache.jackrabbit.oak.run.cli.CommonOptions;
+import org.apache.jackrabbit.oak.run.cli.DocumentBuilderCustomizer;
 import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
 import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
 import org.apache.jackrabbit.oak.run.cli.Options;
 import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.whiteboard.Registration;
 import org.apache.jackrabbit.util.ISO8601;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Collections.emptyMap;
 
 public class IndexCommand implements Command {
     private static final Logger log = 
LoggerFactory.getLogger(IndexCommand.class);
@@ -90,6 +93,7 @@ public class IndexCommand implements Com
                 performReindexInReadWriteMode(indexOpts);
             } else {
                 try (Closer closer = Closer.create()) {
+                    configureCustomizer(opts, closer, true);
                     NodeStoreFixture fixture = 
NodeStoreFixtureProvider.create(opts);
                     closer.register(fixture);
                     execute(fixture, indexOpts, closer);
@@ -237,6 +241,7 @@ public class IndexCommand implements Com
 
     private File performReindexInReadOnlyMode(IndexOptions indexOpts, String 
checkpoint) throws Exception {
         try (Closer closer = Closer.create()) {
+            configureCustomizer(opts, closer, true);
             NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts, 
true);
             closer.register(fixture);
             IndexHelper indexHelper = createIndexHelper(fixture, indexOpts, 
closer);
@@ -261,6 +266,7 @@ public class IndexCommand implements Com
 
     private void connectInReadWriteModeAndImportIndex(IndexOptions indexOpts, 
File indexDir) throws Exception {
         try (Closer closer = Closer.create()) {
+            configureCustomizer(opts, closer, false);
             NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts);
             closer.register(fixture);
             IndexHelper indexHelper = createIndexHelper(fixture, indexOpts, 
closer);
@@ -350,4 +356,15 @@ public class IndexCommand implements Com
     static Path getPath(File file) {
         return file.toPath().normalize().toAbsolutePath();
     }
+
+    private static void configureCustomizer(Options opts, Closer closer, 
boolean readOnlyAccess) {
+        if (opts.getCommonOpts().isDocument()){
+            IndexOptions indexOpts = opts.getOptionBean(IndexOptions.class);
+            if (indexOpts.isReindex()) {
+                IndexDocumentBuilderCustomizer customizer = new 
IndexDocumentBuilderCustomizer(opts, readOnlyAccess);
+                Registration reg = 
opts.getWhiteboard().register(DocumentBuilderCustomizer.class, customizer, 
emptyMap());
+                closer.register(reg::unregister);
+            }
+        }
+    }
 }

Added: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexDocumentBuilderCustomizer.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexDocumentBuilderCustomizer.java?rev=1804642&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexDocumentBuilderCustomizer.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexDocumentBuilderCustomizer.java
 Thu Aug 10 08:41:49 2017
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
+import org.apache.jackrabbit.oak.run.cli.DocumentBuilderCustomizer;
+import org.apache.jackrabbit.oak.run.cli.DocumentNodeStoreOptions;
+import org.apache.jackrabbit.oak.run.cli.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.commons.io.FileUtils.ONE_GB;
+
+class IndexDocumentBuilderCustomizer implements DocumentBuilderCustomizer {
+    private static final String PERSISTENT_CACHE_PROP = 
"oak.documentMK.persCache";
+    private final Logger log = LoggerFactory.getLogger(getClass());
+    private final Options opts;
+    private final DocumentNodeStoreOptions docStoreOpts;
+    private final boolean readOnlyAccess;
+
+    IndexDocumentBuilderCustomizer(Options opts, boolean readOnlyAccess) {
+        this.opts = opts;
+        docStoreOpts = opts.getOptionBean(DocumentNodeStoreOptions.class);
+        this.readOnlyAccess = readOnlyAccess;
+    }
+
+    @Override
+    public void customize(DocumentMK.Builder builder) throws IOException {
+        configurePersistentCache(builder);
+        configureCacheSize(builder);
+
+        if (readOnlyAccess) {
+            configureCacheForReadOnlyMode(builder);
+        }
+    }
+
+    private void configurePersistentCache(DocumentMK.Builder builder) throws 
IOException {
+        if (System.getProperty(PERSISTENT_CACHE_PROP) == null) {
+            File temp = opts.getOptionBean(IndexOptions.class).getWorkDir();
+            File cache = new File(temp, "cache");
+            String cacheConfig = 
String.format("%s,size=4096,binary=0,-nodes,-children", 
cache.getAbsolutePath());
+            builder.setPersistentCache(cacheConfig);
+            log.info("Persistent cache set to [{}]", cacheConfig);
+        }
+    }
+
+    private void configureCacheForReadOnlyMode(DocumentMK.Builder builder) {
+        if (!docStoreOpts.isCacheDistributionDefined()) {
+            builder.memoryCacheDistribution(
+                    35,
+                    10,
+                    15,
+                    2
+            );
+        }
+
+        // usage of this DocumentNodeStore is single threaded. Reduce the
+        // number of cache segments to a minimum. This allows for caching
+        // bigger entries that would otherwise be evicted immediately
+        //TODO Should not be done if later we implement multithreaded indexing
+        builder.setCacheSegmentCount(1);
+        log.info("Configuring cache for single threaded access");
+    }
+
+    private void configureCacheSize(DocumentMK.Builder builder) {
+        //Set cache size to max 4GB or half of min memory
+        if (docStoreOpts.getCacheSize() == 0) {
+            long maxMem = Runtime.getRuntime().maxMemory();
+            long memToUse = Math.min(ONE_GB * 4, maxMem / 2);
+            builder.memoryCacheSize(memToUse);
+            log.info("Initializing cache size to {} ({})", memToUse, 
IOUtils.humanReadableByteCount(memToUse));
+        }
+    }
+}

Propchange: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexDocumentBuilderCustomizer.java
------------------------------------------------------------------------------
    svn:eol-style = native


Reply via email to