Author: thomasm
Date: Wed Jul 22 12:50:35 2020
New Revision: 1880148
URL: http://svn.apache.org/viewvc?rev=1880148&view=rev
Log:
OAK-9147 Config option for NRT queue timeout
Modified:
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/JsonIndexCommand.java
Modified:
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java?rev=1880148&r1=1880147&r2=1880148&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java
(original)
+++
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java
Wed Jul 22 12:50:35 2020
@@ -131,6 +131,7 @@ public class HybridIndexTest extends Abs
private int asyncInterval = Integer.getInteger("asyncInterval", 5);
private int cleanerIntervalInSecs =
Integer.getInteger("cleanerIntervalInSecs", 10);
private int queueSize = Integer.getInteger("queueSize", 1000);
+ private long queueTimeout = Long.getLong("queueTimeout", 100);
private boolean hybridIndexEnabled =
Boolean.getBoolean("hybridIndexEnabled");
private boolean dumpStats = Boolean.getBoolean("dumpStats");
private boolean useOakCodec =
Boolean.parseBoolean(System.getProperty("useOakCodec", "true"));
@@ -385,7 +386,7 @@ public class HybridIndexTest extends Abs
null, //augmentorFactory
mip);
- queue = new DocumentQueue(queueSize, tracker, executorService,
statsProvider);
+ queue = new DocumentQueue(queueSize, queueTimeout, tracker,
executorService, statsProvider);
localIndexObserver = new LocalIndexObserver(queue, statsProvider);
luceneEditorProvider.setIndexingQueue(queue);
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java?rev=1880148&r1=1880147&r2=1880148&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
Wed Jul 22 12:50:35 2020
@@ -237,6 +237,15 @@ public class LuceneIndexProviderService
)
private static final String PROP_HYBRID_QUEUE_SIZE = "hybridQueueSize";
+ public static final long PROP_HYBRID_QUEUE_TIMEOUT_DEFAULT = 100;
+ @Property(
+ longValue = PROP_HYBRID_QUEUE_TIMEOUT_DEFAULT,
+ label = "Queue timeout",
+ description = "Maximum time to wait for adding entries to the
queue used for storing Lucene Documents which need to be " +
+ "added to local index"
+ )
+ private static final String PROP_HYBRID_QUEUE_TIMEOUT =
"hybridQueueTimeout";
+
private static final boolean PROP_DISABLE_DEFN_STORAGE_DEFAULT = false;
@Property(
boolValue = PROP_DISABLE_DEFN_STORAGE_DEFAULT,
@@ -631,7 +640,8 @@ public class LuceneIndexProviderService
}
int queueSize =
PropertiesUtil.toInteger(config.get(PROP_HYBRID_QUEUE_SIZE),
PROP_HYBRID_QUEUE_SIZE_DEFAULT);
- documentQueue = new DocumentQueue(queueSize, tracker,
getExecutorService(), statisticsProvider);
+ long queueOfferTimeoutMillis =
PropertiesUtil.toLong(config.get(PROP_HYBRID_QUEUE_TIMEOUT),
PROP_HYBRID_QUEUE_TIMEOUT_DEFAULT);
+ documentQueue = new DocumentQueue(queueSize, queueOfferTimeoutMillis,
tracker, getExecutorService(), statisticsProvider);
LocalIndexObserver localIndexObserver = new
LocalIndexObserver(documentQueue, statisticsProvider);
regs.add(bundleContext.registerService(Observer.class.getName(),
localIndexObserver, null));
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java?rev=1880148&r1=1880147&r2=1880148&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
Wed Jul 22 12:50:35 2020
@@ -41,6 +41,7 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.commons.concurrent.NotifyingFutureTask;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexNode;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProviderService;
import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
import org.apache.jackrabbit.oak.stats.CounterStats;
import org.apache.jackrabbit.oak.stats.MeterStats;
@@ -70,7 +71,7 @@ public class DocumentQueue implements Cl
* Time in millis for which add call to queue
* would wait before dropping off
*/
- private final int offerTimeMillis;
+ private final long queueOfferTimeoutMillis;
private volatile boolean stopped;
@@ -139,14 +140,15 @@ public class DocumentQueue implements Cl
};
public DocumentQueue(int maxQueueSize, IndexTracker tracker, Executor
executor) {
- this(maxQueueSize, tracker, executor, StatisticsProvider.NOOP);
+ this(maxQueueSize,
LuceneIndexProviderService.PROP_HYBRID_QUEUE_TIMEOUT_DEFAULT,
+ tracker, executor, StatisticsProvider.NOOP);
}
- public DocumentQueue(int maxQueueSize, IndexTracker tracker, Executor
executor, StatisticsProvider sp) {
+ public DocumentQueue(int maxQueueSize, long queueOfferTimeoutMillis,
IndexTracker tracker, Executor executor, StatisticsProvider sp) {
this.docsQueue = new LinkedBlockingDeque<>(maxQueueSize);
this.tracker = tracker;
this.executor = executor;
- this.offerTimeMillis = 100; //Wait for at most 100 mills while adding
stuff to queue
+ this.queueOfferTimeoutMillis = queueOfferTimeoutMillis;
this.queueSizeStats = sp.getCounterStats("HYBRID_QUEUE_SIZE",
StatsOptions.DEFAULT);
this.added = sp.getMeter("HYBRID_ADDED", StatsOptions.DEFAULT);
this.dropped = sp.getMeter("HYBRID_DROPPED", StatsOptions.DEFAULT);
@@ -170,7 +172,7 @@ public class DocumentQueue implements Cl
checkState(!stopped);
boolean added = false;
try {
- added = docsQueue.offer(doc, offerTimeMillis,
TimeUnit.MILLISECONDS);
+ added = docsQueue.offer(doc, queueOfferTimeoutMillis,
TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java?rev=1880148&r1=1880147&r2=1880148&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FrozenNodeRefsByScanningCommand.java
Wed Jul 22 12:50:35 2020
@@ -74,7 +74,7 @@ import joptsimple.OptionSpec;
* Note that any property with uuid that cannot be resolved will *not be
reported*, as that
* is a legitimate use case of uuid property use. Only uuids that resolve will
be analysed.
* <p>
- * Example:
+ * Example:
* <pre>
* java -mx4g -jar oak-run-*.jar frozennoderefsbyscanning
mongodb://localhost/<dbname> -user=admin -password=admin
* </pre>
@@ -248,8 +248,9 @@ public class FrozenNodeRefsByScanningCom
.getExitingScheduledExecutorService((ScheduledThreadPoolExecutor)
Executors.newScheduledThreadPool(5));
StatisticsProvider statsProvider = StatisticsProvider.NOOP;
int queueSize = Integer.getInteger("queueSize", 1000);
+ long queueTimeout = Long.getLong("queueTimeoutMillis", 100);
IndexTracker tracker = new IndexTracker();
- DocumentQueue queue = new DocumentQueue(queueSize, tracker,
executorService, statsProvider);
+ DocumentQueue queue = new DocumentQueue(queueSize, queueTimeout,
tracker, executorService, statsProvider);
ep.setIndexingQueue(queue);
return ep;
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/JsonIndexCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/JsonIndexCommand.java?rev=1880148&r1=1880147&r2=1880148&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/JsonIndexCommand.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/JsonIndexCommand.java
Wed Jul 22 12:50:35 2020
@@ -501,8 +501,9 @@ public class JsonIndexCommand implements
(ScheduledThreadPoolExecutor)
Executors.newScheduledThreadPool(5));
StatisticsProvider statsProvider = StatisticsProvider.NOOP;
int queueSize = Integer.getInteger("queueSize", 1000);
+ long queueTimeout = Long.getLong("queueTimeoutMillis", 100);
IndexTracker tracker = new IndexTracker();
- DocumentQueue queue = new DocumentQueue(queueSize, tracker,
executorService, statsProvider);
+ DocumentQueue queue = new DocumentQueue(queueSize, queueTimeout,
tracker, executorService, statsProvider);
ep.setIndexingQueue(queue);
return ep;
}
