Author: thomasm
Date: Tue Feb 11 13:46:25 2014
New Revision: 1567126
URL: http://svn.apache.org/r1567126
Log:
OAK-1395 Automatically cancel long running queries
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/strategy/ContentMirrorStoreStrategy.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/FilterIterators.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/strategy/ContentMirrorStoreStrategy.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/strategy/ContentMirrorStoreStrategy.java?rev=1567126&r1=1567125&r2=1567126&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/strategy/ContentMirrorStoreStrategy.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/strategy/ContentMirrorStoreStrategy.java
Tue Feb 11 13:46:25 2014
@@ -28,6 +28,7 @@ import org.apache.jackrabbit.oak.api.Pro
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry;
+import org.apache.jackrabbit.oak.query.FilterIterators;
import org.apache.jackrabbit.oak.spi.query.Filter;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -268,6 +269,7 @@ public class ContentMirrorStoreStrategy
readCount++;
if (readCount % 1000 == 0) {
+ FilterIterators.checkReadLimit(readCount);
LOG.warn("Traversed " + readCount + " nodes using
index " + indexName + " with filter " + filter);
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/FilterIterators.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/FilterIterators.java?rev=1567126&r1=1567125&r2=1567126&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/FilterIterators.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/FilterIterators.java
Tue Feb 11 13:46:25 2014
@@ -25,7 +25,52 @@ import java.util.NoSuchElementException;
* or distinct.
*/
public class FilterIterators {
+
+ /**
+ * How many nodes a query may read at most into memory, for "order by" and
+ * "distinct" queries. If this limit is exceeded, the query throws an
+ * exception.
+ */
+ public static final int QUERY_LIMIT_IN_MEMORY =
+ Integer.getInteger("oak.queryLimitInMemory", 10000);
+
+ /**
+ * How many nodes a query may read at most (raw read operations, including
+ * skipped nodes). If this limit is exceeded, the query throws an
exception.
+ */
+ public static final int QUERY_LIMIT_READS =
+ Integer.getInteger("oak.queryLimitReads", 100000);
+
+ /**
+ * Verify the number of in-memory nodes is below the limit.
+ *
+ * @param count the number of nodes
+ * @throws UnsupportedOperationException if the limit was exceeded
+ */
+ public static void checkMemoryLimit(long count) {
+ if (count > QUERY_LIMIT_IN_MEMORY) {
+ throw new UnsupportedOperationException(
+ "The query read more than " +
+ QUERY_LIMIT_IN_MEMORY + " nodes in memory. " +
+ "To avoid running out of memory, processing was
stopped.");
+ }
+ }
+ /**
+ * Verify the number of node read operations is below the limit.
+ *
+ * @param count the number of read operations
+ * @throws UnsupportedOperationException if the limit was exceeded
+ */
+ public static void checkReadLimit(long count) {
+ if (count > QUERY_LIMIT_READS) {
+ throw new UnsupportedOperationException(
+ "The query read or traversed more than " +
+ QUERY_LIMIT_READS + " nodes. " +
+ "To avoid affecting other tasks, processing was
stopped.");
+ }
+ }
+
public static <K> Iterator<K> newCombinedFilter(
Iterator<K> it, boolean distinct, long limit, long offset,
Comparator<K> orderBy) {
@@ -90,6 +135,7 @@ public class FilterIterators {
while (source.hasNext()) {
current = source.next();
if (distinctSet.add(current)) {
+ checkMemoryLimit(distinctSet.size());
return;
}
}
@@ -153,6 +199,7 @@ public class FilterIterators {
while (source.hasNext()) {
K x = source.next();
list.add(x);
+ checkMemoryLimit(list.size());
// from time to time, sort and truncate
// this should results in O(n*log(2*keep)) operations,
// which is close to the optimum O(n*log(keep))
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java?rev=1567126&r1=1567125&r2=1567126&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/Cursors.java
Tue Feb 11 13:46:25 2014
@@ -24,6 +24,7 @@ import javax.annotation.Nullable;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry;
+import org.apache.jackrabbit.oak.query.FilterIterators;
import org.apache.jackrabbit.oak.query.index.IndexRowImpl;
import org.apache.jackrabbit.oak.spi.query.Filter.PathRestriction;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
@@ -114,6 +115,9 @@ public class Cursors {
}
+ /**
+ * This class allows to iterate over the parent nodes of the wrapped
cursor.
+ */
private static class AncestorCursor extends PathCursor {
public AncestorCursor(Cursor cursor, int level) {
@@ -161,6 +165,7 @@ public class Cursors {
@Override
public boolean apply(@Nullable String input) {
+ FilterIterators.checkMemoryLimit(known.size());
// Set.add returns true for new entries
return known.add(input);
}
@@ -289,6 +294,7 @@ public class Cursors {
readCount++;
if (readCount % 1000 == 0) {
+ FilterIterators.checkReadLimit(readCount);
LOG.warn("Traversed " + readCount + " nodes with
filter " + filter + "; consider creating an index or changing the query");
}
