Author: reschke
Date: Tue Dec 8 14:44:53 2015
New Revision: 1718626
URL: http://svn.apache.org/viewvc?rev=1718626&view=rev
Log:
OAK-3729: RDBDocumentStore: implement RDB-specific VersionGC support for lookup
of deleted documents
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
(with props)
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java?rev=1718626&r1=1718625&r2=1718626&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java
Tue Dec 8 14:44:53 2015
@@ -60,6 +60,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.document.rdb.RDBBlobStore;
import org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore;
import org.apache.jackrabbit.oak.plugins.document.rdb.RDBOptions;
+import org.apache.jackrabbit.oak.plugins.document.rdb.RDBVersionGCSupport;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
import org.apache.jackrabbit.oak.plugins.document.util.RevisionsKey;
import org.apache.jackrabbit.oak.plugins.document.util.StringValue;
@@ -881,6 +882,8 @@ public class DocumentMK {
DocumentStore store = getDocumentStore();
if (store instanceof MongoDocumentStore) {
return new MongoVersionGCSupport((MongoDocumentStore) store);
+ } else if (store instanceof RDBDocumentStore) {
+ return new RDBVersionGCSupport((RDBDocumentStore) store);
} else {
return new VersionGCSupport(store);
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java?rev=1718626&r1=1718625&r2=1718626&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
Tue Dec 8 14:44:53 2015
@@ -234,7 +234,17 @@ public class RDBDocumentStore implements
@Override
public <T extends Document> List<T> query(Collection<T> collection, String
fromKey, String toKey, String indexedProperty,
long startValue, int limit) {
- return internalQuery(collection, fromKey, toKey, indexedProperty,
startValue, limit);
+ List<QueryCondition> conditions = Collections.emptyList();
+ if (indexedProperty != null) {
+ conditions = Collections.singletonList(new
QueryCondition(indexedProperty, ">=", startValue));
+ }
+ return internalQuery(collection, fromKey, toKey, conditions, limit);
+ }
+
+ @Nonnull
+ protected <T extends Document> List<T> query(Collection<T> collection,
String fromKey, String toKey,
+ List<QueryCondition> conditions, int limit) {
+ return internalQuery(collection, fromKey, toKey, conditions, limit);
}
@Override
@@ -1204,14 +1214,16 @@ public class RDBDocumentStore implements
private Map<Thread, QueryContext> qmap = new ConcurrentHashMap<Thread,
QueryContext>();
private <T extends Document> List<T> internalQuery(Collection<T>
collection, String fromKey, String toKey,
- String indexedProperty, long startValue, int limit) {
+ List<QueryCondition> conditions, int limit) {
Connection connection = null;
RDBTableMetaData tmd = getTable(collection);
- if (indexedProperty != null &&
(!INDEXEDPROPERTIES.contains(indexedProperty))) {
- String message = "indexed property " + indexedProperty + " not
supported, query was '>= '" + startValue
- + "'; supported properties are " + INDEXEDPROPERTIES;
- LOG.info(message);
- throw new DocumentStoreException(message);
+ for (QueryCondition cond : conditions) {
+ if (!INDEXEDPROPERTIES.contains(cond.getPropertyName())) {
+ String message = "indexed property " + cond.getPropertyName()
+ " not supported, query was '" + cond.getOperator()
+ + "'" + cond.getValue() + "'; supported properties are
" + INDEXEDPROPERTIES;
+ LOG.info(message);
+ throw new DocumentStoreException(message);
+ }
}
try {
long now = System.currentTimeMillis();
@@ -1223,7 +1235,7 @@ public class RDBDocumentStore implements
connection = this.ch.getROConnection();
String from = collection == Collection.NODES &&
NodeDocument.MIN_ID_VALUE.equals(fromKey) ? null : fromKey;
String to = collection == Collection.NODES &&
NodeDocument.MAX_ID_VALUE.equals(toKey) ? null : toKey;
- List<RDBRow> dbresult = db.query(connection, tmd, from, to,
indexedProperty, startValue, limit);
+ List<RDBRow> dbresult = db.query(connection, tmd, from, to,
conditions, limit);
connection.commit();
int size = dbresult.size();
@@ -1668,4 +1680,34 @@ public class RDBDocumentStore implements
protected NodeDocumentCache getNodeDocumentCache() {
return nodesCache;
}
+
+ // slightly extended query support
+ protected static class QueryCondition {
+
+ private final String propertyName, operator;
+ private final long value;
+
+ public QueryCondition(String propertyName, String operator, long
value) {
+ this.propertyName = propertyName;
+ this.operator = operator;
+ this.value = value;
+ }
+
+ public String getPropertyName() {
+ return propertyName;
+ }
+
+ public String getOperator() {
+ return operator;
+ }
+
+ public long getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s %s %d", propertyName, operator, value);
+ }
+ }
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java?rev=1718626&r1=1718625&r2=1718626&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
Tue Dec 8 14:44:53 2015
@@ -29,9 +29,13 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
@@ -41,6 +45,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Condition;
import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Key;
+import
org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.QueryCondition;
import
org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.RDBTableMetaData;
import
org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStoreDB.FETCHFIRSTSYNTAX;
import
org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStoreDB.PreparedStatementComponent;
@@ -318,9 +323,29 @@ public class RDBDocumentStoreJDBC {
}
}
+ private final static Map<String, String> INDEXED_PROP_MAPPING;
+ static {
+ Map<String, String> tmp = new HashMap<String, String>();
+ tmp.put(MODIFIED, "MODIFIED");
+ tmp.put(NodeDocument.HAS_BINARY_FLAG, "HASBINARY");
+ tmp.put(NodeDocument.DELETED_ONCE, "DELETEDONCE");
+ INDEXED_PROP_MAPPING = Collections.unmodifiableMap(tmp);
+ }
+
+ private final static Set<String> SUPPORTED_OPS;
+ static {
+ Set<String> tmp = new HashSet<String>();
+ tmp.add(">=");
+ tmp.add(">");
+ tmp.add("<=");
+ tmp.add("<");
+ tmp.add("=");
+ SUPPORTED_OPS = Collections.unmodifiableSet(tmp);
+ }
+
@Nonnull
- public List<RDBRow> query(Connection connection, RDBTableMetaData tmd,
String minId, String maxId, String indexedProperty,
- long startValue, int limit) throws SQLException {
+ public List<RDBRow> query(Connection connection, RDBTableMetaData tmd,
String minId, String maxId,
+ List<QueryCondition> conditions, int limit) throws SQLException {
long start = System.currentTimeMillis();
StringBuilder selectClause = new StringBuilder();
StringBuilder whereClause = new StringBuilder();
@@ -339,20 +364,16 @@ public class RDBDocumentStoreJDBC {
whereClause.append(whereSep).append("ID < ?");
whereSep = " and ";
}
-
- if (indexedProperty != null) {
- if (MODIFIED.equals(indexedProperty)) {
- whereClause.append(whereSep).append("MODIFIED >= ?");
- } else if (NodeDocument.HAS_BINARY_FLAG.equals(indexedProperty)) {
- if (startValue != NodeDocument.HAS_BINARY_VAL) {
- throw new DocumentStoreException("unsupported value for
property " + NodeDocument.HAS_BINARY_FLAG);
- }
- whereClause.append(whereSep).append("HASBINARY = 1");
- } else if (NodeDocument.DELETED_ONCE.equals(indexedProperty)) {
- if (startValue != 1) {
- throw new DocumentStoreException("unsupported value for
property " + NodeDocument.DELETED_ONCE);
- }
- whereClause.append(whereSep).append("DELETEDONCE = 1");
+ for (QueryCondition cond : conditions) {
+ String op = cond.getOperator();
+ if (!SUPPORTED_OPS.contains(op)) {
+ throw new DocumentStoreException("unsupported operator: " +
op);
+ }
+ String indexedProperty = cond.getPropertyName();
+ String column = INDEXED_PROP_MAPPING.get(indexedProperty);
+ if (column != null) {
+ whereClause.append(whereSep).append(column).append("
").append(op).append(" ?");
+ whereSep = " and ";
} else {
throw new DocumentStoreException("unsupported indexed
property: " + indexedProperty);
}
@@ -390,9 +411,8 @@ public class RDBDocumentStoreJDBC {
if (maxId != null) {
setIdInStatement(tmd, stmt, si++, maxId);
}
-
- if (MODIFIED.equals(indexedProperty)) {
- stmt.setLong(si++, startValue);
+ for (QueryCondition cond : conditions) {
+ stmt.setLong(si++, cond.getValue());
}
if (limit != Integer.MAX_VALUE) {
stmt.setFetchSize(limit);
@@ -423,14 +443,13 @@ public class RDBDocumentStoreJDBC {
long elapsed = System.currentTimeMillis() - start;
if (this.queryHitsLimit != 0 && result.size() > this.queryHitsLimit) {
String message = String.format(
- "Potentially excessive query with %d hits (limited to %d,
configured QUERYHITSLIMIT %d), elapsed time %dms, params minid '%s' maxid '%s'
indexedProperty %s startValue %d limit %d. Check calling method.",
- result.size(), limit, this.queryHitsLimit, elapsed, minId,
maxId, indexedProperty, startValue, limit);
+ "Potentially excessive query with %d hits (limited to %d,
configured QUERYHITSLIMIT %d), elapsed time %dms, params minid '%s' maxid '%s'
condition %s limit %d. Check calling method.",
+ result.size(), limit, this.queryHitsLimit, elapsed, minId,
maxId, conditions, limit);
LOG.info(message, new Exception("call stack"));
} else if (this.queryTimeLimit != 0 && elapsed > this.queryTimeLimit) {
String message = String.format(
- "Long running query with %d hits (limited to %d), elapsed
time %dms (configured QUERYTIMELIMIT %d), params minid '%s' maxid '%s'
indexedProperty %s startValue %d limit %d. Read %d chars from DATA and %d bytes
from BDATA. Check calling method.",
- result.size(), limit, elapsed, this.queryTimeLimit, minId,
maxId, indexedProperty, startValue, limit, dataTotal,
- bdataTotal);
+ "Long running query with %d hits (limited to %d), elapsed
time %dms (configured QUERYTIMELIMIT %d), params minid '%s' maxid '%s'
condition %s limit %d. Read %d chars from DATA and %d bytes from BDATA. Check
calling method.",
+ result.size(), limit, elapsed, this.queryTimeLimit, minId,
maxId, conditions, limit, dataTotal, bdataTotal);
LOG.info(message, new Exception("call stack"));
}
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java?rev=1718626&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
Tue Dec 8 14:44:53 2015
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document.rdb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.plugins.document.Collection;
+import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
+import org.apache.jackrabbit.oak.plugins.document.VersionGCSupport;
+import
org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.QueryCondition;
+
+import com.google.common.collect.AbstractIterator;
+
+/**
+ * RDB specific version of {@link VersionGCSupport} which uses an extended
query
+ * interface to fetch required {@link NodeDocuments}.
+ */
+public class RDBVersionGCSupport extends VersionGCSupport {
+ private RDBDocumentStore store;
+
+ public RDBVersionGCSupport(RDBDocumentStore store) {
+ super(store);
+ this.store = store;
+ }
+
+ @Override
+ public Iterable<NodeDocument> getPossiblyDeletedDocs(final long
lastModifiedTime) {
+ List<QueryCondition> conditions = new ArrayList<QueryCondition>();
+ conditions.add(new QueryCondition(NodeDocument.DELETED_ONCE, "=", 1));
+ conditions.add(new QueryCondition(NodeDocument.MODIFIED_IN_SECS, "<",
NodeDocument.getModifiedInSecs(lastModifiedTime)));
+ return getIterator(conditions);
+ }
+
+ private Iterable<NodeDocument> getIterator(final List<QueryCondition>
conditions) {
+ return new Iterable<NodeDocument>() {
+ @Override
+ public Iterator<NodeDocument> iterator() {
+ return new AbstractIterator<NodeDocument>() {
+
+ private static final int BATCH_SIZE = 100;
+ private String startId = NodeDocument.MIN_ID_VALUE;
+ private Iterator<NodeDocument> batch = nextBatch();
+
+ @Override
+ protected NodeDocument computeNext() {
+ // read next batch if necessary
+ if (!batch.hasNext()) {
+ batch = nextBatch();
+ }
+
+ NodeDocument doc;
+ if (batch.hasNext()) {
+ doc = batch.next();
+ // remember current id
+ startId = doc.getId();
+ } else {
+ doc = endOfData();
+ }
+ return doc;
+ }
+
+ private Iterator<NodeDocument> nextBatch() {
+ List<NodeDocument> result =
store.query(Collection.NODES, startId, NodeDocument.MAX_ID_VALUE, conditions,
+ BATCH_SIZE);
+ return result.iterator();
+ }
+ };
+ }
+ };
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java?rev=1718626&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java
Tue Dec 8 14:44:53 2015
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document.rdb;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.plugins.document.AbstractDocumentStoreTest;
+import org.apache.jackrabbit.oak.plugins.document.Collection;
+import org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture;
+import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
+import org.apache.jackrabbit.oak.plugins.document.UpdateOp;
+import
org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.QueryCondition;
+import org.junit.Test;
+
+public class RDBDocumentStoreTest extends AbstractDocumentStoreTest {
+
+ public RDBDocumentStoreTest(DocumentStoreFixture dsf) {
+ super(dsf);
+ }
+
+ @Test
+ public void testRDBQuery() {
+ if (ds instanceof RDBDocumentStore) {
+ RDBDocumentStore rds = (RDBDocumentStore) ds;
+ // create ten documents
+ long now = System.currentTimeMillis();
+ String base = this.getClass().getName() + ".testRDBQuery-";
+ for (int i = 0; i < 10; i++) {
+ String id = base + i;
+ UpdateOp up = new UpdateOp(id, true);
+ up.set("_id", id);
+ up.set(NodeDocument.DELETED_ONCE, i % 2 == 1);
+ up.set(NodeDocument.MODIFIED_IN_SECS, now++);
+ boolean success = super.ds.create(Collection.NODES,
Collections.singletonList(up));
+ assertTrue("document with " + id + " not created", success);
+ removeMe.add(id);
+ }
+
+ List<QueryCondition> conditions = new ArrayList<QueryCondition>();
+ // matches every second
+ conditions.add(new QueryCondition(NodeDocument.DELETED_ONCE, "=",
0));
+ // matches first eight
+ conditions.add(new QueryCondition(NodeDocument.MODIFIED_IN_SECS,
"<", now - 2));
+ List<NodeDocument> result = rds.query(Collection.NODES, base, base
+ "A", conditions, 10);
+ assertEquals(4, result.size());
+ }
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java
------------------------------------------------------------------------------
svn:eol-style = native
