Author: reschke
Date: Wed Dec 9 16:17:06 2015
New Revision: 1718895
URL: http://svn.apache.org/viewvc?rev=1718895&view=rev
Log:
OAK-3730: RDBDocumentStore: implement RDB-specific VersionGC support for lookup
of split documents
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java?rev=1718895&r1=1718894&r2=1718895&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
Wed Dec 9 16:17:06 2015
@@ -238,13 +238,13 @@ public class RDBDocumentStore implements
if (indexedProperty != null) {
conditions = Collections.singletonList(new
QueryCondition(indexedProperty, ">=", startValue));
}
- return internalQuery(collection, fromKey, toKey, conditions, limit);
+ return internalQuery(collection, fromKey, toKey, EMPTY_KEY_PATTERN,
conditions, limit);
}
@Nonnull
protected <T extends Document> List<T> query(Collection<T> collection,
String fromKey, String toKey,
- List<QueryCondition> conditions, int limit) {
- return internalQuery(collection, fromKey, toKey, conditions, limit);
+ List<String> excludeKeyPatterns, List<QueryCondition> conditions,
int limit) {
+ return internalQuery(collection, fromKey, toKey, excludeKeyPatterns,
conditions, limit);
}
@Override
@@ -512,6 +512,8 @@ public class RDBDocumentStore implements
// utility class for performing low-level operations
private RDBDocumentStoreJDBC db;
+ protected static final List<String> EMPTY_KEY_PATTERN =
Collections.emptyList();
+
private Map<String, String> metadata;
// set of supported indexed properties
@@ -1214,7 +1216,7 @@ public class RDBDocumentStore implements
private Map<Thread, QueryContext> qmap = new ConcurrentHashMap<Thread,
QueryContext>();
private <T extends Document> List<T> internalQuery(Collection<T>
collection, String fromKey, String toKey,
- List<QueryCondition> conditions, int limit) {
+ List<String> excludeKeyPatterns, List<QueryCondition> conditions,
int limit) {
Connection connection = null;
RDBTableMetaData tmd = getTable(collection);
for (QueryCondition cond : conditions) {
@@ -1235,7 +1237,7 @@ public class RDBDocumentStore implements
connection = this.ch.getROConnection();
String from = collection == Collection.NODES &&
NodeDocument.MIN_ID_VALUE.equals(fromKey) ? null : fromKey;
String to = collection == Collection.NODES &&
NodeDocument.MAX_ID_VALUE.equals(toKey) ? null : toKey;
- List<RDBRow> dbresult = db.query(connection, tmd, from, to,
conditions, limit);
+ List<RDBRow> dbresult = db.query(connection, tmd, from, to,
excludeKeyPatterns, conditions, limit);
connection.commit();
int size = dbresult.size();
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java?rev=1718895&r1=1718894&r2=1718895&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
Wed Dec 9 16:17:06 2015
@@ -345,7 +345,7 @@ public class RDBDocumentStoreJDBC {
@Nonnull
public List<RDBRow> query(Connection connection, RDBTableMetaData tmd,
String minId, String maxId,
- List<QueryCondition> conditions, int limit) throws SQLException {
+ List<String> excludeKeyPatterns, List<QueryCondition> conditions,
int limit) throws SQLException {
long start = System.currentTimeMillis();
StringBuilder selectClause = new StringBuilder();
StringBuilder whereClause = new StringBuilder();
@@ -364,6 +364,16 @@ public class RDBDocumentStoreJDBC {
whereClause.append(whereSep).append("ID < ?");
whereSep = " and ";
}
+ if (!excludeKeyPatterns.isEmpty()) {
+ whereClause.append(whereSep);
+ whereSep = " and ";
+ whereClause.append("not (");
+ for (int i = 0; i < excludeKeyPatterns.size(); i++) {
+ whereClause.append(i == 0 ? "" : " or ");
+ whereClause.append("ID like ?");
+ }
+ whereClause.append(")");
+ }
for (QueryCondition cond : conditions) {
String op = cond.getOperator();
if (!SUPPORTED_OPS.contains(op)) {
@@ -411,6 +421,9 @@ public class RDBDocumentStoreJDBC {
if (maxId != null) {
setIdInStatement(tmd, stmt, si++, maxId);
}
+ for (String keyPattern : excludeKeyPatterns) {
+ setIdInStatement(tmd, stmt, si++, keyPattern);
+ }
for (QueryCondition cond : conditions) {
stmt.setLong(si++, cond.getValue());
}
@@ -443,13 +456,14 @@ public class RDBDocumentStoreJDBC {
long elapsed = System.currentTimeMillis() - start;
if (this.queryHitsLimit != 0 && result.size() > this.queryHitsLimit) {
String message = String.format(
- "Potentially excessive query with %d hits (limited to %d,
configured QUERYHITSLIMIT %d), elapsed time %dms, params minid '%s' maxid '%s'
condition %s limit %d. Check calling method.",
- result.size(), limit, this.queryHitsLimit, elapsed, minId,
maxId, conditions, limit);
+ "Potentially excessive query with %d hits (limited to %d,
configured QUERYHITSLIMIT %d), elapsed time %dms, params minid '%s' maxid '%s'
excludeKeyPatterns %s condition %s limit %d. Check calling method.",
+ result.size(), limit, this.queryHitsLimit, elapsed, minId,
maxId, excludeKeyPatterns, conditions, limit);
LOG.info(message, new Exception("call stack"));
} else if (this.queryTimeLimit != 0 && elapsed > this.queryTimeLimit) {
String message = String.format(
- "Long running query with %d hits (limited to %d), elapsed
time %dms (configured QUERYTIMELIMIT %d), params minid '%s' maxid '%s'
condition %s limit %d. Read %d chars from DATA and %d bytes from BDATA. Check
calling method.",
- result.size(), limit, elapsed, this.queryTimeLimit, minId,
maxId, conditions, limit, dataTotal, bdataTotal);
+ "Long running query with %d hits (limited to %d), elapsed
time %dms (configured QUERYTIMELIMIT %d), params minid '%s' maxid '%s'
excludeKeyPatterns %s conditions %s limit %d. Read %d chars from DATA and %d
bytes from BDATA. Check calling method.",
+ result.size(), limit, elapsed, this.queryTimeLimit, minId,
maxId, excludeKeyPatterns, conditions, limit,
+ dataTotal, bdataTotal);
LOG.info(message, new Exception("call stack"));
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java?rev=1718895&r1=1718894&r2=1718895&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
Wed Dec 9 16:17:06 2015
@@ -16,15 +16,22 @@
*/
package org.apache.jackrabbit.oak.plugins.document.rdb;
+import static com.google.common.collect.Iterables.filter;
+
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import org.apache.jackrabbit.oak.plugins.document.Collection;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
+import org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType;
import org.apache.jackrabbit.oak.plugins.document.VersionGCSupport;
import
org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.QueryCondition;
+import com.google.common.base.Predicate;
import com.google.common.collect.AbstractIterator;
/**
@@ -44,10 +51,28 @@ public class RDBVersionGCSupport extends
List<QueryCondition> conditions = new ArrayList<QueryCondition>();
conditions.add(new QueryCondition(NodeDocument.DELETED_ONCE, "=", 1));
conditions.add(new QueryCondition(NodeDocument.MODIFIED_IN_SECS, "<",
NodeDocument.getModifiedInSecs(lastModifiedTime)));
- return getIterator(conditions);
+ return getIterator(RDBDocumentStore.EMPTY_KEY_PATTERN, conditions);
+ }
+
+ private Iterable<NodeDocument> getSplitDocuments() {
+ List<QueryCondition> conditions = Collections.emptyList();
+ // absent support for SDTYPE as indexed property: exclude those
+ // documents from the query which definitively aren't split documents
+ List<String> excludeKeyPatterns = Arrays.asList("_:/%", "__:/%",
"___:/%");
+ return getIterator(excludeKeyPatterns, conditions);
+ }
+
+ @Override
+ protected Iterable<NodeDocument> identifyGarbage(final Set<SplitDocType>
gcTypes, final long oldestRevTimeStamp) {
+ return filter(getSplitDocuments(), new Predicate<NodeDocument>() {
+ @Override
+ public boolean apply(NodeDocument doc) {
+ return gcTypes.contains(doc.getSplitDocType()) &&
doc.hasAllRevisionLessThan(oldestRevTimeStamp);
+ }
+ });
}
- private Iterable<NodeDocument> getIterator(final List<QueryCondition>
conditions) {
+ private Iterable<NodeDocument> getIterator(final List<String>
excludeKeyPatterns, final List<QueryCondition> conditions) {
return new Iterable<NodeDocument>() {
@Override
public Iterator<NodeDocument> iterator() {
@@ -76,8 +101,8 @@ public class RDBVersionGCSupport extends
}
private Iterator<NodeDocument> nextBatch() {
- List<NodeDocument> result =
store.query(Collection.NODES, startId, NodeDocument.MAX_ID_VALUE, conditions,
- BATCH_SIZE);
+ List<NodeDocument> result =
store.query(Collection.NODES, startId, NodeDocument.MAX_ID_VALUE,
+ excludeKeyPatterns, conditions, BATCH_SIZE);
return result.iterator();
}
};
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java?rev=1718895&r1=1718894&r2=1718895&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java
Wed Dec 9 16:17:06 2015
@@ -18,9 +18,9 @@ package org.apache.jackrabbit.oak.plugin
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -39,7 +39,7 @@ public class RDBDocumentStoreTest extend
}
@Test
- public void testRDBQuery() {
+ public void testRDBQueryConditions() {
if (ds instanceof RDBDocumentStore) {
RDBDocumentStore rds = (RDBDocumentStore) ds;
// create ten documents
@@ -61,8 +61,37 @@ public class RDBDocumentStoreTest extend
conditions.add(new QueryCondition(NodeDocument.DELETED_ONCE, "=",
0));
// matches first eight
conditions.add(new QueryCondition(NodeDocument.MODIFIED_IN_SECS,
"<", now - 2));
- List<NodeDocument> result = rds.query(Collection.NODES, base, base
+ "A", conditions, 10);
+ List<NodeDocument> result = rds.query(Collection.NODES, base, base
+ "A", RDBDocumentStore.EMPTY_KEY_PATTERN,
+ conditions, 10);
assertEquals(4, result.size());
}
}
+
+ @Test
+ public void testRDBQueryKeyPatterns() {
+ if (ds instanceof RDBDocumentStore) {
+ RDBDocumentStore rds = (RDBDocumentStore) ds;
+ // create ten documents
+ String base = this.getClass().getName() + ".testRDBQuery-";
+ for (int i = 0; i < 10; i++) {
+ // every second is a "regular" path
+ String id = "1:" + (i % 2 == 1 ? "p" : "") + "/" + base + i;
+ UpdateOp up = new UpdateOp(id, true);
+ up.set("_id", id);
+ up.set("_test", base);
+ boolean success = super.ds.create(Collection.NODES,
Collections.singletonList(up));
+ assertTrue("document with " + id + " not created", success);
+ removeMe.add(id);
+ }
+ System.out.println(removeMe);
+ List<QueryCondition> conditions = new ArrayList<QueryCondition>();
+ List<NodeDocument> result = rds.query(Collection.NODES,
NodeDocument.MIN_ID_VALUE, NodeDocument.MAX_ID_VALUE,
+ Arrays.asList("_:/%", "__:/%", "___:/%"), conditions,
10000);
+ for (NodeDocument d : result) {
+ if (base.equals(d.get("_test"))) {
+ assertTrue(d.getId().startsWith("1:p"));
+ }
+ }
+ }
+ }
}
