Repository: incubator-rya Updated Branches: refs/heads/master 2564ac0ab -> 1d99212e6
RYA-313. Closes #196. Aggregation now is performed over batches of 1000. Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/1d99212e Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/1d99212e Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/1d99212e Branch: refs/heads/master Commit: 1d99212e60ac1ac0a0c8fef227e5c959c6826aa1 Parents: 2564ac0 Author: isper3at <smith...@gmail.com> Authored: Thu Aug 3 19:03:11 2017 -0400 Committer: Caleb Meier <caleb.me...@parsons.com> Committed: Sun Aug 6 19:54:39 2017 -0700 ---------------------------------------------------------------------- .../apache/rya/mongodb/MongoDBQueryEngine.java | 29 +++++-- .../operators/aggregation/AggregationUtil.java | 14 ++-- .../aggregation/PipelineOperators.java | 23 +----- .../operators/aggregation/SetOperators.java | 87 ++++---------------- .../aggregation/VariableOperators.java | 37 ++------- .../operators/query/ArrayOperators.java | 24 ++---- .../operators/query/ComparisonOperators.java | 47 ++--------- .../operators/query/ConditionalOperators.java | 69 +++------------- .../operators/query/LogicalOperators.java | 55 +++---------- .../RyaStatementBindingSetCursorIterator.java | 33 ++++---- .../iter/RyaStatementCursorIterator.java | 33 ++++---- 11 files changed, 133 insertions(+), 318 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBQueryEngine.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBQueryEngine.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBQueryEngine.java index 2b49e61..8932fc4 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBQueryEngine.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBQueryEngine.java @@ -20,7 +20,6 @@ package org.apache.rya.mongodb; import static com.google.common.base.Preconditions.checkNotNull; -import java.io.Closeable; import java.io.IOException; import java.util.Collection; import java.util.HashSet; @@ -39,6 +38,7 @@ import org.apache.rya.mongodb.iter.NonCloseableRyaStatementCursorIterator; import org.apache.rya.mongodb.iter.RyaStatementBindingSetCursorIterator; import org.apache.rya.mongodb.iter.RyaStatementCursorIterable; import org.apache.rya.mongodb.iter.RyaStatementCursorIterator; +import org.bson.Document; import org.calrissian.mango.collect.CloseableIterable; import org.openrdf.query.BindingSet; @@ -48,6 +48,8 @@ import com.mongodb.DB; import com.mongodb.DBCollection; import com.mongodb.DBObject; import com.mongodb.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; import info.aduna.iteration.CloseableIteration; @@ -91,13 +93,17 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio final Set<DBObject> queries = new HashSet<DBObject>(); final DBObject query = strategy.getQuery(stmt); queries.add(query); - final RyaStatementCursorIterator iterator = new RyaStatementCursorIterator(coll, queries, strategy, conf.getAuthorizations()); + final MongoDatabase db = mongoClient.getDatabase(conf.getMongoDBName()); + final MongoCollection<Document> collection = db.getCollection(conf.getTriplesCollectionName()); + final RyaStatementCursorIterator iterator = new RyaStatementCursorIterator(collection, queries, strategy, + conf.getAuthorizations()); if (maxResults != null) { iterator.setMaxResults(maxResults); } return iterator; } + @Override public CloseableIteration<? extends Entry<RyaStatement, BindingSet>, RyaDAOException> queryWithBindingSet( final Collection<Entry<RyaStatement, BindingSet>> stmts, @@ -118,7 +124,8 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio } // TODO not sure what to do about regex ranges? - final RyaStatementBindingSetCursorIterator iterator = new RyaStatementBindingSetCursorIterator(coll, rangeMap, strategy, conf.getAuthorizations()); + final RyaStatementBindingSetCursorIterator iterator = new RyaStatementBindingSetCursorIterator( + getCollection(conf), rangeMap, strategy, conf.getAuthorizations()); if (maxResults != null) { iterator.setMaxResults(maxResults); @@ -145,7 +152,8 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio } // TODO not sure what to do about regex ranges? - final RyaStatementCursorIterator iterator = new RyaStatementCursorIterator(coll, queries, strategy, configuration.getAuthorizations()); + final RyaStatementCursorIterator iterator = new RyaStatementCursorIterator(getCollection(conf), queries, + strategy, configuration.getAuthorizations()); if (maxResults != null) { iterator.setMaxResults(maxResults); @@ -166,7 +174,9 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio // TODO not sure what to do about regex ranges? // TODO this is gross - final RyaStatementCursorIterable iterator = new RyaStatementCursorIterable(new NonCloseableRyaStatementCursorIterator(new RyaStatementCursorIterator(coll, queries, strategy, configuration.getAuthorizations()))); + final RyaStatementCursorIterable iterator = new RyaStatementCursorIterable( + new NonCloseableRyaStatementCursorIterator(new RyaStatementCursorIterator(getCollection(getConf()), + queries, strategy, configuration.getAuthorizations()))); return iterator; } catch (final Exception e) { @@ -185,7 +195,9 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio // TODO not sure what to do about regex ranges? // TODO this is gross - final RyaStatementCursorIterable iterator = new RyaStatementCursorIterable(new NonCloseableRyaStatementCursorIterator(new RyaStatementCursorIterator(coll, queries, strategy, configuration.getAuthorizations()))); + final RyaStatementCursorIterable iterator = new RyaStatementCursorIterable( + new NonCloseableRyaStatementCursorIterator(new RyaStatementCursorIterator(getCollection(getConf()), + queries, strategy, configuration.getAuthorizations()))); return iterator; } catch (final Exception e) { @@ -193,6 +205,11 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio } } + private MongoCollection getCollection(final MongoDBRdfConfiguration conf) { + final MongoDatabase db = mongoClient.getDatabase(conf.getMongoDBName()); + return db.getCollection(conf.getTriplesCollectionName()); + } + @Override public void close() throws IOException { // if (mongoClient != null){ mongoClient.close(); } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/AggregationUtil.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/AggregationUtil.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/AggregationUtil.java index 6e4f250..082d7bd 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/AggregationUtil.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/AggregationUtil.java @@ -34,9 +34,9 @@ import org.apache.rya.mongodb.MongoDbRdfConstants; import org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy; import org.apache.rya.mongodb.document.operators.aggregation.PipelineOperators.RedactAggregationResult; import org.apache.rya.mongodb.document.util.AuthorizationsUtil; +import org.bson.Document; import com.google.common.collect.Lists; -import com.mongodb.BasicDBObject; import com.mongodb.DBObject; /** @@ -58,7 +58,7 @@ public final class AggregationUtil { * @return the {@link List} of {@link DBObject}s that represents the $redact * aggregation pipeline. */ - public static List<DBObject> createRedactPipeline(final Authorizations authorizations) { + public static List<Document> createRedactPipeline(final Authorizations authorizations) { if (MongoDbRdfConstants.ALL_AUTHORIZATIONS.equals(authorizations)) { return Lists.newArrayList(); } @@ -69,29 +69,29 @@ public final class AggregationUtil { final String mapVariableCursorName = "dvItemCursorTag"; - final BasicDBObject anyElementTrue = + final Document anyElementTrue = anyElementTrue( map( documentVisibilityField, mapVariableCursorName, setIsSubsetNullSafe( "$$" + mapVariableCursorName, - authList.toArray() + authList ) ) ); // If the field is empty then there are no authorizations required, // so all users should be able to view it when they query. - final BasicDBObject isFieldSizeZero = + final Document isFieldSizeZero = eq( size(documentVisibilityField), 0 ); - final BasicDBObject orExpression = or(anyElementTrue, isFieldSizeZero); + final Document orExpression = or(anyElementTrue, isFieldSizeZero); - final List<DBObject> pipeline = new ArrayList<>(); + final List<Document> pipeline = new ArrayList<>(); pipeline.add( redact( orExpression, http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/PipelineOperators.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/PipelineOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/PipelineOperators.java index b73a6c7..2d65312 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/PipelineOperators.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/PipelineOperators.java @@ -21,6 +21,8 @@ package org.apache.rya.mongodb.document.operators.aggregation; import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.rya.mongodb.document.operators.query.ConditionalOperators.cond; +import org.bson.Document; + import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObjectBuilder; @@ -83,26 +85,9 @@ public final class PipelineOperators { * the expression passes. * @param rejectResult the {@link RedactAggregationResult} to return when * the expression fails. - * @return the $redact expression {@link BasicDBObject}. - */ - public static BasicDBObject redact(final BasicDBObject expression, final RedactAggregationResult acceptResult, final RedactAggregationResult rejectResult) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) redact(builder, expression, acceptResult, rejectResult).get(); - } - - /** - * Creates a $redact expression. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) - * @param expression the expression to run redact on. - * @param acceptResult the {@link RedactAggregationResult} to return when - * the expression passes. - * @param rejectResult the {@link RedactAggregationResult} to return when - * the expression fails. * @return the $redact expression {@link BasicDBObjectBuilder}. */ - public static BasicDBObjectBuilder redact(final BasicDBObjectBuilder builder, final BasicDBObject expression, final RedactAggregationResult acceptResult, final RedactAggregationResult rejectResult) { - checkNotNull(builder); - builder.add("$redact", cond(expression, acceptResult.toString(), rejectResult.toString())); - return builder; + public static Document redact(final Document expression, final RedactAggregationResult acceptResult, final RedactAggregationResult rejectResult) { + return new Document("$redact", cond(expression, acceptResult.toString(), rejectResult.toString())); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/SetOperators.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/SetOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/SetOperators.java index d5b8738..9ef8204 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/SetOperators.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/SetOperators.java @@ -18,15 +18,13 @@ */ package org.apache.rya.mongodb.document.operators.aggregation; -import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.rya.mongodb.document.operators.query.ConditionalOperators.ifNull; import java.util.Arrays; import java.util.Collections; +import java.util.List; -import com.mongodb.BasicDBObject; -import com.mongodb.BasicDBObjectBuilder; -import com.mongodb.DBObject; +import org.bson.Document; /** * Utility methods for MongoDB set operators. @@ -42,48 +40,20 @@ public final class SetOperators { * Checks if the field intersects the set. * @param field the field to check. * @param set the set to check. - * @return the $setIntersection expression {@link BasicDBObject}. + * @return the $setIntersection expression {@link Document}. */ - public static BasicDBObject setIntersection(final String field, final Object[] set) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) setIntersection(builder, field, set).get(); - } - - /** - * Checks if the field intersects the set. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) - * @param field the field to check. - * @param set the set to check. - * @return the $setIntersection expression {@link BasicDBObjectBuilder}. - */ - public static BasicDBObjectBuilder setIntersection(final BasicDBObjectBuilder builder, final String field, final Object[] set) { - checkNotNull(builder); - builder.add("$setIntersection", Arrays.asList(field, set)); - return builder; - } - - /** - * Checks if the expression is a subset of the set. - * @param expression the expression to see if it's in the set. - * @param set the set to check against. - * @return the $setIsSubset expression {@link BasicDBObject}. - */ - public static BasicDBObject setIsSubset(final DBObject expression, final Object[] set) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) setIsSubset(builder, expression, set).get(); + public static Document setIntersection(final String field, final List<String> set) { + return new Document("$setIntersection", Arrays.asList(field, set)); } /** * Checks if the expression is a subset of the set. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) * @param expression the expression to see if it's in the set. * @param set the set to check against. - * @return the $setIsSubset expression {@link BasicDBObjectBuilder}. + * @return the $setIsSubset expression {@link Document}. */ - public static BasicDBObjectBuilder setIsSubset(final BasicDBObjectBuilder builder, final DBObject expression, final Object[] set) { - checkNotNull(builder); - builder.add("$setIsSubset", Arrays.asList(expression, set).toArray(new Object[0])); - return builder; + public static Document setIsSubset(final Document expression, final List<String> set) { + return new Document("$setIsSubset", Arrays.asList(expression, set)); } /** @@ -91,25 +61,11 @@ public final class SetOperators { * null. * @param field the field to see if it's in the set. * @param set the set to check against. - * @return the $setIsSubset expression {@link BasicDBObject}. + * @return the $setIsSubset expression {@link Document}. */ - public static BasicDBObject setIsSubsetNullSafe(final String field, final Object[] set) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) setIsSubsetNullSafe(builder, field, set).get(); - } - - /** - * Checks if the field is a subset of the set and is safe if the field is - * null. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) - * @param field the field to see if it's in the set. - * @param set the set to check against. - * @return the $setIsSubset expression {@link BasicDBObjectBuilder}. - */ - public static BasicDBObjectBuilder setIsSubsetNullSafe(final BasicDBObjectBuilder builder, final String field, final Object[] set) { - checkNotNull(builder); - final Object emptyAccess = Collections.emptyList().toArray(); - return setIsSubset(builder, + public static Document setIsSubsetNullSafe(final String field, final List<String> set) { + final Object emptyAccess = Collections.emptyList(); + return setIsSubset( ifNull( field, emptyAccess @@ -121,22 +77,9 @@ public final class SetOperators { /** * Checks if any elements from the expression are {@code true}. * @param expression the expression to see if any elements are {@code true}. - * @return the $anyElementTrue expression {@link BasicDBObject}. - */ - public static BasicDBObject anyElementTrue(final DBObject expression) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) anyElementTrue(builder, expression).get(); - } - - /** - * Checks if any elements from the expression are {@code true}. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) - * @param expression the expression to see if any elements are {@code true}. - * @return the $anyElementTrue expression {@link BasicDBObjectBuilder}. + * @return the $anyElementTrue expression {@link Document}. */ - public static BasicDBObjectBuilder anyElementTrue(final BasicDBObjectBuilder builder, final DBObject expression) { - checkNotNull(builder); - builder.add("$anyElementTrue", expression); - return builder; + public static Document anyElementTrue(final Document expression) { + return new Document("$anyElementTrue", expression); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/VariableOperators.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/VariableOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/VariableOperators.java index 1fcf8e3..e014dc3 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/VariableOperators.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/VariableOperators.java @@ -18,11 +18,9 @@ */ package org.apache.rya.mongodb.document.operators.aggregation; -import static com.google.common.base.Preconditions.checkNotNull; +import javax.xml.parsers.DocumentBuilder; -import com.mongodb.BasicDBObject; -import com.mongodb.BasicDBObjectBuilder; -import com.mongodb.DBObject; +import org.bson.Document; /** * Utility methods for variable operators. @@ -43,31 +41,14 @@ public final class VariableOperators { * by this variable. * @param in the expression to apply to each item in the {@code input} * array. The expression accesses the item by its variable name. - * @return the $map expression {@link BasicDBObject}. + * @return the $map expression {@link DocumentBuilder}. */ - public static BasicDBObject map(final String input, final String as, final DBObject in) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) map(builder, input, as, in).get(); - } + public static Document map(final String input, final String as, final Document in) { + final Document mapDoc = new Document() + .append("input", input) + .append("as", as) + .append("in", in); - /** - * Applies an expression to each item in an array and returns an array with - * the applied results. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) - * @param input an expression that resolves to an array. - * @param as the variable name for the items in the {@code input} array. - * The {@code in} expression accesses each item in the {@code input} array - * by this variable. - * @param in the expression to apply to each item in the {@code input} - * array. The expression accesses the item by its variable name. - * @return the $map expression {@link BasicDBObjectBuilder}. - */ - public static BasicDBObjectBuilder map(final BasicDBObjectBuilder builder, final String input, final String as, final DBObject in) { - checkNotNull(builder); - builder.push("$map") - .add("input", input) - .add("as", as) - .add("in", in); - return builder; + return new Document("$map", mapDoc); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ArrayOperators.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ArrayOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ArrayOperators.java index 1413ce8..b3daa7d 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ArrayOperators.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ArrayOperators.java @@ -18,10 +18,9 @@ */ package org.apache.rya.mongodb.document.operators.query; -import static com.google.common.base.Preconditions.checkNotNull; +import javax.xml.parsers.DocumentBuilder; -import com.mongodb.BasicDBObject; -import com.mongodb.BasicDBObjectBuilder; +import org.bson.Document; /** * Utility methods for array operators. @@ -36,22 +35,9 @@ public final class ArrayOperators { /** * Creates an $size MongoDB expression. * @param expression the expression to get the size of. - * @return the $size expression {@link BasicDBObject}. + * @return the $size expression {@link DocumentBuilder}. */ - public static BasicDBObject size(final Object expression) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) size(builder, expression).get(); - } - - /** - * Creates an $size MongoDB expression. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) - * @param expression the expression to get the size of. - * @return the $size expression {@link BasicDBObjectBuilder}. - */ - public static BasicDBObjectBuilder size(final BasicDBObjectBuilder builder, final Object expression) { - checkNotNull(builder); - builder.add("$size", expression); - return builder; + public static Document size(final Object expression) { + return new Document("$size", expression); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ComparisonOperators.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ComparisonOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ComparisonOperators.java index 596493a..5187401 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ComparisonOperators.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ComparisonOperators.java @@ -18,12 +18,9 @@ */ package org.apache.rya.mongodb.document.operators.query; -import static com.google.common.base.Preconditions.checkNotNull; - import java.util.Arrays; -import com.mongodb.BasicDBObject; -import com.mongodb.BasicDBObjectBuilder; +import org.bson.Document; /** * Utility methods for comparison operators. @@ -37,49 +34,23 @@ public final class ComparisonOperators { /** * Creates a $gt MongoDB expression. + * * @param expression the expression. * @param value the value to test if the expression is greater than - * @return the $gt expression {@link BasicDBObject}. - */ - public static BasicDBObject gt(final BasicDBObject expression, final Number value) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) gt(builder, expression, value).get(); - } - - /** - * Creates a $gt MongoDB expression. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) - * @param expression the expression. - * @param value the value to test if the expression is greater than - * @return the $gt expression {@link BasicDBObjectBuilder}. - */ - public static BasicDBObjectBuilder gt(final BasicDBObjectBuilder builder, final BasicDBObject expression, final Number value) { - checkNotNull(builder); - builder.add("$gt", Arrays.asList(expression, value)); - return builder; - } - - /** - * Creates an $eq MongoDB expression. - * @param expression1 the first expression. - * @param expression2 the second expression. - * @return the $eq expression {@link BasicDBObject}. + * @return the $gt expression {@link Document}. */ - public static BasicDBObject eq(final BasicDBObject expression1, final Object expression2) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) eq(builder, expression1, expression2).get(); + public static Document gt(final Document expression, final Number value) { + return new Document("$gt", Arrays.asList(expression, value)); } /** * Creates an $eq MongoDB expression. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) + * * @param expression1 the first expression. * @param expression2 the second expression. - * @return the $eq expression {@link BasicDBObjectBuilder}. + * @return the $eq expression {@link Document}. */ - public static BasicDBObjectBuilder eq(final BasicDBObjectBuilder builder, final BasicDBObject expression1, final Object expression2) { - checkNotNull(builder); - builder.add("$eq", Arrays.asList(expression1, expression2)); - return builder; + public static Document eq(final Document expression1, final Object expression2) { + return new Document("$eq", Arrays.asList(expression1, expression2)); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ConditionalOperators.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ConditionalOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ConditionalOperators.java index c976519..dd83762 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ConditionalOperators.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ConditionalOperators.java @@ -18,10 +18,10 @@ */ package org.apache.rya.mongodb.document.operators.query; -import static com.google.common.base.Preconditions.checkNotNull; - import java.util.Arrays; +import org.bson.Document; + import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObjectBuilder; @@ -42,74 +42,27 @@ public final class ConditionalOperators { * {@code ifStatement} is {@code true}. * @param elseResult the {@link Object} to return when the * {@code ifStatement} is {@code false}. - * @return the "if" expression {@link BasicDBObject}. - */ - public static BasicDBObject ifThenElse(final BasicDBObject ifStatement, final Object thenResult, final Object elseResult) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) ifThenElse(builder, ifStatement, thenResult, elseResult).get(); - } - - /** - * Creates an "if-then-else" MongoDB expression. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) - * @param ifStatement the "if" statement {@link BasicDBObject}. - * @param thenResult the {@link Object} to return when the - * {@code ifStatement} is {@code true}. - * @param elseResult the {@link Object} to return when the - * {@code ifStatement} is {@code false}. * @return the "if" expression {@link BasicDBObjectBuilder}. */ - public static BasicDBObjectBuilder ifThenElse(final BasicDBObjectBuilder builder, final BasicDBObject ifStatement, final Object thenResult, final Object elseResult) { - checkNotNull(builder); - builder.add("if", ifStatement); - builder.append("then", thenResult); - builder.append("else", elseResult); - return builder; - } - - /** - * Checks if the expression is {@code null} and replaces it if it is. - * @param expression the expression to {@code null} check. - * @param replacementExpression the expression to replace it with if it's - * {@code null}. - * @return the $ifNull expression {@link BasicDBObject}. - */ - public static BasicDBObject ifNull(final Object expression, final Object replacementExpression) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) ifNull(builder, expression, replacementExpression).get(); + public static Document ifThenElse(final Document ifStatement, final Object thenResult, final Object elseResult) { + return new Document("if", ifStatement) + .append("then", thenResult) + .append("else", elseResult); } /** * Checks if the expression is {@code null} and replaces it if it is. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) * @param expression the expression to {@code null} check. * @param replacementExpression the expression to replace it with if it's * {@code null}. * @return the $ifNull expression {@link BasicDBObjectBuilder}. */ - public static BasicDBObjectBuilder ifNull(final BasicDBObjectBuilder builder, final Object expression, final Object replacementExpression) { - checkNotNull(builder); - builder.add("$ifNull", Arrays.asList(expression, replacementExpression)); - return builder; - } - - /** - * Creates an "$cond" MongoDB expression. - * @param expression the expression {@link BasicDBObject}. - * @param thenResult the {@link Object} to return when the - * {@code expression} is {@code true}. - * @param elseResult the {@link Object} to return when the - * {@code expression} is {@code false}. - * @return the $cond expression {@link BasicDBObject}. - */ - public static BasicDBObject cond(final BasicDBObject expression, final Object thenResult, final Object elseResult) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) cond(builder, expression, thenResult, elseResult).get(); + public static Document ifNull(final Object expression, final Object replacementExpression) { + return new Document("$ifNull", Arrays.asList(expression, replacementExpression)); } /** * Creates an "$cond" MongoDB expression. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) * @param expression the expression {@link BasicDBObject}. * @param thenResult the {@link Object} to return when the * {@code expression} is {@code true}. @@ -117,9 +70,7 @@ public final class ConditionalOperators { * {@code expression} is {@code false}. * @return the $cond expression {@link BasicDBObjectBuilder}. */ - public static BasicDBObjectBuilder cond(final BasicDBObjectBuilder builder, final BasicDBObject expression, final Object thenResult, final Object elseResult) { - checkNotNull(builder); - builder.add("$cond", ifThenElse(expression, thenResult, elseResult)); - return builder; + public static Document cond(final Document expression, final Object thenResult, final Object elseResult) { + return new Document("$cond", ifThenElse(expression, thenResult, elseResult)); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/LogicalOperators.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/LogicalOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/LogicalOperators.java index 07dae18..a9f7595 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/LogicalOperators.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/LogicalOperators.java @@ -18,13 +18,13 @@ */ package org.apache.rya.mongodb.document.operators.query; -import static com.google.common.base.Preconditions.checkNotNull; - import java.util.List; +import javax.xml.parsers.DocumentBuilder; + +import org.bson.Document; + import com.google.common.collect.Lists; -import com.mongodb.BasicDBObject; -import com.mongodb.BasicDBObjectBuilder; /** * Utility methods for logical operators. @@ -41,62 +41,33 @@ public final class LogicalOperators { * @param lhs the left-hand side operand. * @param rhs the right-hand side operand. * @param extras any additional operands. - * @return the $and expression {@link BasicDBObject}. + * @return the $and expression {@link DocumentBuilder}. */ - public static BasicDBObject and(final BasicDBObject lhs, final BasicDBObject rhs, final BasicDBObject... extras) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) and(builder, lhs, rhs, extras).get(); - } - - /** - * Creates an $and MongoDB expression. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) - * @param lhs the left-hand side operand. - * @param rhs the right-hand side operand. - * @param extras any additional operands. - * @return the $and expression {@link BasicDBObjectBuilder}. - */ - public static BasicDBObjectBuilder and(final BasicDBObjectBuilder builder, final BasicDBObject lhs, final BasicDBObject rhs, final BasicDBObject... extras) { - checkNotNull(builder); - final List<BasicDBObject> operands = Lists.newArrayList(lhs, rhs); + public static Document and(final Document lhs, final Document rhs, final Document... extras) { + final List<Document> operands = Lists.newArrayList(lhs, rhs); if (extras != null && extras.length > 0) { operands.addAll(Lists.newArrayList(extras)); } - builder.add("$and", operands); - return builder; - } - - /** - * Creates an $or MongoDB expression. - * @param lhs the left-hand side operand. - * @param rhs the right-hand side operand. - * @param extras any additional operands. - * @return the $or expression {@link BasicDBObject}. - */ - public static BasicDBObject or(final BasicDBObject lhs, final BasicDBObject rhs, final BasicDBObject... extras) { - final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); - return (BasicDBObject) or(builder, lhs, rhs, extras).get(); + return new Document("$and", operands); } /** * Creates an $or MongoDB expression. - * @param builder the {@link BasicDBObjectBuilder}. (not {@code null}) + * @param builder the {@link DocumentBuilder}. (not {@code null}) * @param lhs the left-hand side operand. * @param rhs the right-hand side operand. * @param extras any additional operands. - * @return the $or expression {@link BasicDBObjectBuilder}. + * @return the $or expression {@link DocumentBuilder}. */ - public static BasicDBObjectBuilder or(final BasicDBObjectBuilder builder, final BasicDBObject lhs, final BasicDBObject rhs, final BasicDBObject... extras) { - checkNotNull(builder); - final List<BasicDBObject> operands = Lists.newArrayList(lhs, rhs); + public static Document or(final Document lhs, final Document rhs, final Document... extras) { + final List<Document> operands = Lists.newArrayList(lhs, rhs); if (extras != null && extras.length > 0) { operands.addAll(Lists.newArrayList(extras)); } - builder.add("$or", operands); - return builder; + return new Document("$or", operands); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java index 4807c4f..18f71d2 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java @@ -31,35 +31,37 @@ import org.apache.rya.api.domain.RyaStatement; import org.apache.rya.api.persist.RyaDAOException; import org.apache.rya.mongodb.dao.MongoDBStorageStrategy; import org.apache.rya.mongodb.document.operators.aggregation.AggregationUtil; +import org.bson.Document; import org.openrdf.query.BindingSet; import com.google.common.collect.Multimap; -import com.mongodb.AggregationOutput; -import com.mongodb.BasicDBObject; -import com.mongodb.DBCollection; import com.mongodb.DBObject; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.util.JSON; import info.aduna.iteration.CloseableIteration; public class RyaStatementBindingSetCursorIterator implements CloseableIteration<Entry<RyaStatement, BindingSet>, RyaDAOException> { private static final Logger log = Logger.getLogger(RyaStatementBindingSetCursorIterator.class); - private final DBCollection coll; + private final MongoCollection<Document> coll; private final Multimap<DBObject, BindingSet> rangeMap; private final Iterator<DBObject> queryIterator; private Long maxResults; - private Iterator<DBObject> resultsIterator; + private Iterator<Document> resultsIterator; private RyaStatement currentStatement; private Collection<BindingSet> currentBindingSetCollection; private Iterator<BindingSet> currentBindingSetIterator; private final MongoDBStorageStrategy<RyaStatement> strategy; private final Authorizations auths; - public RyaStatementBindingSetCursorIterator(final DBCollection coll, - final Multimap<DBObject, BindingSet> rangeMap, final MongoDBStorageStrategy<RyaStatement> strategy, final Authorizations auths) { + public RyaStatementBindingSetCursorIterator(final MongoCollection<Document> coll, + final Multimap<DBObject, BindingSet> rangeMap, final MongoDBStorageStrategy<RyaStatement> strategy, + final Authorizations auths) { this.coll = coll; this.rangeMap = rangeMap; - this.queryIterator = rangeMap.keySet().iterator(); + queryIterator = rangeMap.keySet().iterator(); this.strategy = strategy; this.auths = auths; } @@ -94,8 +96,9 @@ public class RyaStatementBindingSetCursorIterator implements CloseableIteration< } if (currentResultCursorIsValid()) { // convert to Rya Statement - final DBObject queryResult = resultsIterator.next(); - currentStatement = strategy.deserializeDBObject(queryResult); + final Document queryResult = resultsIterator.next(); + final DBObject dbo = (DBObject) JSON.parse(queryResult.toJson()); + currentStatement = strategy.deserializeDBObject(dbo); currentBindingSetIterator = currentBindingSetCollection.iterator(); } } @@ -106,12 +109,14 @@ public class RyaStatementBindingSetCursorIterator implements CloseableIteration< currentBindingSetCollection = rangeMap.get(currentQuery); // Executing redact aggregation to only return documents the user // has access to. - final List<DBObject> pipeline = new ArrayList<>(); - pipeline.add(new BasicDBObject("$match", currentQuery)); + final List<Document> pipeline = new ArrayList<>(); + pipeline.add(new Document("$match", currentQuery)); pipeline.addAll(AggregationUtil.createRedactPipeline(auths)); log.debug(pipeline); - final AggregationOutput output = coll.aggregate(pipeline); - resultsIterator = output.results().iterator(); + + final AggregateIterable<Document> aggIter = coll.aggregate(pipeline); + aggIter.batchSize(1000); + resultsIterator = aggIter.iterator(); if (resultsIterator.hasNext()) { break; } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementCursorIterator.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementCursorIterator.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementCursorIterator.java index 2f6fd44..1a5eb99 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementCursorIterator.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementCursorIterator.java @@ -29,27 +29,29 @@ import org.apache.rya.api.domain.RyaStatement; import org.apache.rya.api.persist.RyaDAOException; import org.apache.rya.mongodb.dao.MongoDBStorageStrategy; import org.apache.rya.mongodb.document.operators.aggregation.AggregationUtil; +import org.bson.Document; -import com.mongodb.AggregationOutput; -import com.mongodb.BasicDBObject; -import com.mongodb.DBCollection; import com.mongodb.DBObject; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.util.JSON; import info.aduna.iteration.CloseableIteration; public class RyaStatementCursorIterator implements CloseableIteration<RyaStatement, RyaDAOException> { private static final Logger log = Logger.getLogger(RyaStatementCursorIterator.class); - private final DBCollection coll; + private final MongoCollection coll; private final Iterator<DBObject> queryIterator; - private Iterator<DBObject> resultsIterator; + private Iterator<Document> resultsIterator; private final MongoDBStorageStrategy<RyaStatement> strategy; private Long maxResults; private final Authorizations auths; - public RyaStatementCursorIterator(final DBCollection coll, final Set<DBObject> queries, final MongoDBStorageStrategy<RyaStatement> strategy, final Authorizations auths) { - this.coll = coll; - this.queryIterator = queries.iterator(); + public RyaStatementCursorIterator(final MongoCollection<Document> collection, final Set<DBObject> queries, + final MongoDBStorageStrategy<RyaStatement> strategy, final Authorizations auths) { + coll = collection; + queryIterator = queries.iterator(); this.strategy = strategy; this.auths = auths; } @@ -69,8 +71,9 @@ public class RyaStatementCursorIterator implements CloseableIteration<RyaStateme } if (currentCursorIsValid()) { // convert to Rya Statement - final DBObject queryResult = resultsIterator.next(); - final RyaStatement statement = strategy.deserializeDBObject(queryResult); + final Document queryResult = resultsIterator.next(); + final DBObject dbo = (DBObject) JSON.parse(queryResult.toJson()); + final RyaStatement statement = strategy.deserializeDBObject(dbo); return statement; } return null; @@ -82,12 +85,14 @@ public class RyaStatementCursorIterator implements CloseableIteration<RyaStateme // Executing redact aggregation to only return documents the user // has access to. - final List<DBObject> pipeline = new ArrayList<>(); - pipeline.add(new BasicDBObject("$match", currentQuery)); + final List<Document> pipeline = new ArrayList<>(); + pipeline.add(new Document("$match", currentQuery)); pipeline.addAll(AggregationUtil.createRedactPipeline(auths)); log.debug(pipeline); - final AggregationOutput output = coll.aggregate(pipeline); - resultsIterator = output.results().iterator(); + final AggregateIterable<Document> output = coll.aggregate(pipeline); + output.batchSize(1000); + + resultsIterator = output.iterator(); if (resultsIterator.hasNext()) { break; }