Repository: incubator-rya Updated Branches: refs/heads/master 45efa55b0 -> c8989858e
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/11349b11/extras/indexing/src/test/java/org/apache/rya/indexing/statement/metadata/StatementMetadataOptimizerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/org/apache/rya/indexing/statement/metadata/StatementMetadataOptimizerTest.java b/extras/indexing/src/test/java/org/apache/rya/indexing/statement/metadata/StatementMetadataOptimizerTest.java new file mode 100644 index 0000000..5a1cef8 --- /dev/null +++ b/extras/indexing/src/test/java/org/apache/rya/indexing/statement/metadata/StatementMetadataOptimizerTest.java @@ -0,0 +1,162 @@ +package org.apache.rya.indexing.statement.metadata; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.rya.accumulo.AccumuloRdfConfiguration; +import org.apache.rya.api.RdfCloudTripleStoreConfiguration; +import org.apache.rya.api.domain.RyaURI; +import org.apache.rya.indexing.accumulo.ConfigUtils; +import org.apache.rya.indexing.external.matching.JoinSegment; +import org.apache.rya.indexing.statement.metadata.matching.StatementMetadataExternalSetProvider; +import org.apache.rya.indexing.statement.metadata.matching.StatementMetadataNode; +import org.apache.rya.indexing.statement.metadata.matching.StatementMetadataOptimizer; +import org.apache.rya.mongodb.MongoDBRdfConfiguration; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import org.openrdf.query.MalformedQueryException; +import org.openrdf.query.algebra.Filter; +import org.openrdf.query.algebra.QueryModelNode; +import org.openrdf.query.algebra.StatementPattern; +import org.openrdf.query.algebra.TupleExpr; +import org.openrdf.query.algebra.ValueExpr; +import org.openrdf.query.algebra.helpers.StatementPatternCollector; +import org.openrdf.query.parser.ParsedQuery; +import org.openrdf.query.parser.sparql.SPARQLParser; + +@RunWith(value = Parameterized.class) +public class StatementMetadataOptimizerTest { + + private static final String query1 = "prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> select ?x ?y where {_:blankNode rdf:type rdf:Statement; rdf:subject <http://Joe>; " + + "rdf:predicate <http://worksAt>; rdf:object ?x; <http://createdBy> ?y; <http://createdOn> \'2017-01-04\'^^xsd:date }"; + private static final String query2 = "prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> select ?x ?y where {_:blankNode2 rdf:type rdf:Statement; rdf:subject <http://Bob>; " + + "rdf:predicate <http://worksAt>; rdf:object ?x; <http://createdBy> ?y; <http://createdOn> \'2017-02-04\'^^xsd:date }"; + private static final String query3 = "prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> select ?x ?y where {_:blankNode2 rdf:type rdf:Statement; rdf:subject <http://Frank>; " + + "rdf:predicate <http://worksAt>; rdf:object ?x; <http://createdBy> ?y; <http://createdOn> \'2017-03-04\'^^xsd:date }"; + private static final String query4 = "prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> select ?x ?y where {_:blankNode1 rdf:type rdf:Statement; rdf:subject <http://Joe>; " + + "rdf:predicate <http://worksAt>; rdf:object ?x; <http://createdBy> ?y; <http://createdOn> \'2017-01-04\'^^xsd:date. " + + "_:blankNode2 rdf:type rdf:Statement; rdf:subject <http://Bob>; " + + "rdf:predicate <http://worksAt>; rdf:object ?x; <http://createdBy> ?y; <http://createdOn> \'2017-02-04\'^^xsd:date }"; + private static final String query5 = "prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> select ?x ?y where {_:blankNode1 rdf:type rdf:Statement; rdf:subject <http://Joe>; " + + "rdf:predicate <http://worksAt>; rdf:object ?x; <http://createdBy> ?y; <http://createdOn> \'2017-01-04\'^^xsd:date. " + + "_:blankNode2 rdf:type rdf:Statement; rdf:subject <http://Bob>; " + + "rdf:predicate <http://worksAt>; rdf:object ?a; <http://createdBy> ?b; <http://createdOn> \'2017-02-04\'^^xsd:date. " + + "OPTIONAL{ _:blankNode3 rdf:type rdf:Statement; rdf:subject <http://Frank>; " + + "rdf:predicate <http://worksAt>; rdf:object ?c; <http://createdBy> <http://Doug>; <http://createdOn> \'2017-03-04\'^^xsd:date } }"; + private static final String query6 = "prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> select ?x ?y where {?m rdf:type rdf:Statement; rdf:subject <http://Joe>; " + + "rdf:predicate <http://worksAt>; rdf:object ?x; <http://createdBy> ?y; <http://createdOn> \'2017-01-04\'^^xsd:date. " + + "{ ?o rdf:type rdf:Statement; rdf:subject <http://Frank>; " + + "rdf:predicate <http://worksAt>; rdf:object ?c; <http://createdBy> ?p; <http://createdOn> \'2017-03-04\'^^xsd:date . } " + + "UNION {?n rdf:predicate <http://worksAt>; rdf:object ?a; <http://createdBy> ?c; <http://createdOn> \'2017-02-04\'^^xsd:date; " + + "rdf:type rdf:Statement; rdf:subject ?p. } }"; + + private String query; + private Set<StatementMetadataNode<?>> expected; + private static SPARQLParser parser = new SPARQLParser(); + private StatementMetadataOptimizer mongoOptimizer; + private StatementMetadataOptimizer accumuloOptimizer; + + public StatementMetadataOptimizerTest(String query, Set<StatementMetadataNode<?>> expected) { + this.query = query; + this.expected = expected; + } + + @Before + public void init() { + RdfCloudTripleStoreConfiguration mongoConf = (RdfCloudTripleStoreConfiguration) getConf(true); + RdfCloudTripleStoreConfiguration accumuloConf = (RdfCloudTripleStoreConfiguration) getConf(false); + mongoOptimizer = new StatementMetadataOptimizer(mongoConf); + accumuloOptimizer = new StatementMetadataOptimizer(accumuloConf); + } + + @Parameters + public static Collection<Object[]> data() throws MalformedQueryException { + return Arrays.asList(new Object[][] { { query1, getExpected(query1) }, { query2, getExpected(query2) }, + { query3, getExpected(query3) }, { query4, getExpected(query4) }, { query5, getExpected(query5) }, + { query6, getExpected(query6) } }); + } + + @Test + public void testAccumuloMatchQuery() throws MalformedQueryException { + ParsedQuery pq = parser.parseQuery(query, null); + TupleExpr te = pq.getTupleExpr(); + System.out.println("Parametrized query is : " + te); + accumuloOptimizer.optimize(te, null, null); + System.out.println("Result of optimization is : " + te); + assertEquals(expected, StatementMetadataTestUtils.getMetadataNodes(te)); + } + + @Test + public void testMongoMatchQuery() throws MalformedQueryException { + SPARQLParser parser = new SPARQLParser(); + ParsedQuery pq = parser.parseQuery(query, null); + TupleExpr te = pq.getTupleExpr(); + System.out.println("Parametrized query is : " + te); + mongoOptimizer.optimize(te, null, null); + System.out.println("Result of optimization is : " + te); + assertEquals(expected, StatementMetadataTestUtils.getMetadataNodes(te)); + } + + private static RdfCloudTripleStoreConfiguration getConf(boolean useMongo) { + + RdfCloudTripleStoreConfiguration conf; + Set<RyaURI> propertySet = new HashSet<RyaURI>( + Arrays.asList(new RyaURI("http://createdBy";), new RyaURI("http://createdOn";))); + if (useMongo) { + MongoDBRdfConfiguration mConf = new MongoDBRdfConfiguration(); + mConf.setBoolean("sc.useMongo", true); + mConf.setMongoInstance("localhost"); + mConf.setMongoPort("27017"); + mConf.setMongoDBName("rya_"); + conf = mConf; + } else { + conf = new AccumuloRdfConfiguration(); + conf.setBoolean(ConfigUtils.USE_MOCK_INSTANCE, true); + conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, "rya_"); + conf.set(ConfigUtils.CLOUDBASE_USER, "root"); + conf.set(ConfigUtils.CLOUDBASE_PASSWORD, ""); + conf.set(ConfigUtils.CLOUDBASE_INSTANCE, "instance"); + conf.set(ConfigUtils.CLOUDBASE_AUTHS, ""); + } + conf.setStatementMetadataProperties(propertySet); + return conf; + } + + private static Set<StatementMetadataNode<?>> getExpected(String query) throws MalformedQueryException { + ParsedQuery pq = parser.parseQuery(query, null); + StatementMetadataExternalSetProvider provider = new StatementMetadataExternalSetProvider( + (RdfCloudTripleStoreConfiguration) getConf(false)); + List<StatementPattern> patterns = StatementPatternCollector.process(pq.getTupleExpr()); + JoinSegment<StatementMetadataNode<?>> segment = new JoinSegment<StatementMetadataNode<?>>( + new HashSet<QueryModelNode>(patterns), new ArrayList<QueryModelNode>(patterns), + new HashMap<ValueExpr, Filter>()); + return new HashSet<>(provider.getExternalSets(segment)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/11349b11/extras/indexing/src/test/java/org/apache/rya/indexing/statement/metadata/StatementMetadataTestUtils.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/org/apache/rya/indexing/statement/metadata/StatementMetadataTestUtils.java b/extras/indexing/src/test/java/org/apache/rya/indexing/statement/metadata/StatementMetadataTestUtils.java new file mode 100644 index 0000000..de499e7 --- /dev/null +++ b/extras/indexing/src/test/java/org/apache/rya/indexing/statement/metadata/StatementMetadataTestUtils.java @@ -0,0 +1,100 @@ +package org.apache.rya.indexing.statement.metadata; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.rya.api.domain.RyaURI; +import org.openrdf.model.URI; +import org.openrdf.model.Value; +import org.openrdf.model.vocabulary.RDF; +import org.openrdf.query.algebra.QueryModelNode; +import org.openrdf.query.algebra.StatementPattern; +import org.openrdf.query.algebra.TupleExpr; +import org.openrdf.query.algebra.Var; +import org.openrdf.query.algebra.evaluation.impl.ExternalSet; +import org.openrdf.query.algebra.helpers.QueryModelVisitorBase; + +public class StatementMetadataTestUtils { + + private static final List<RyaURI> uriList = Arrays.asList(new RyaURI(RDF.TYPE.toString()), + new RyaURI(RDF.SUBJECT.toString()), new RyaURI(RDF.PREDICATE.toString()), + new RyaURI(RDF.OBJECT.toString())); + + public static Set<QueryModelNode> getMetadataNodes(TupleExpr query) { + MetadataNodeCollector collector = new MetadataNodeCollector(); + query.visit(collector); + return collector.getNodes(); + } + + public static class MetadataNodeCollector extends QueryModelVisitorBase<RuntimeException> { + + Set<QueryModelNode> qNodes = new HashSet<>(); + + @Override + public void meetNode(final QueryModelNode node) { + if (node instanceof ExternalSet) { + qNodes.add(node); + } + super.meetNode(node); + } + + public Set<QueryModelNode> getNodes() { + return qNodes; + } + } + + public static Set<StatementPattern> getMetadataStatementPatterns(TupleExpr te, Set<RyaURI> properties) { + MetadataStatementPatternCollector collector = new MetadataStatementPatternCollector(properties); + te.visit(collector); + return collector.getNodes(); + + } + + public static class MetadataStatementPatternCollector extends QueryModelVisitorBase<RuntimeException> { + + private Set<StatementPattern> nodes; + private Set<RyaURI> properties; + + public MetadataStatementPatternCollector(Set<RyaURI> properties) { + this.properties = properties; + nodes = new HashSet<>(); + } + + @Override + public void meet(StatementPattern node) { + Var predicate = node.getPredicateVar(); + Value val = predicate.getValue(); + if (val != null && val instanceof URI) { + RyaURI ryaVal = new RyaURI(val.stringValue()); + if (uriList.contains(ryaVal) || properties.contains(ryaVal)) { + nodes.add(node); + } + } + super.meet(node); + } + + public Set<StatementPattern> getNodes() { + return nodes; + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/11349b11/extras/rya.benchmark/src/main/java/org/apache/rya/benchmark/query/PCJOptimizerBenchmark.java ---------------------------------------------------------------------- diff --git a/extras/rya.benchmark/src/main/java/org/apache/rya/benchmark/query/PCJOptimizerBenchmark.java b/extras/rya.benchmark/src/main/java/org/apache/rya/benchmark/query/PCJOptimizerBenchmark.java index fd74e8b..efbe745 100644 --- a/extras/rya.benchmark/src/main/java/org/apache/rya/benchmark/query/PCJOptimizerBenchmark.java +++ b/extras/rya.benchmark/src/main/java/org/apache/rya/benchmark/query/PCJOptimizerBenchmark.java @@ -116,7 +116,7 @@ public class PCJOptimizerBenchmark { private final Map<BenchmarkParams, BenchmarkValues> unchainedBenchmarkValues = new HashMap<>(); @Setup - public void buildBenchmarkValues() throws MalformedQueryException { + public void buildBenchmarkValues() throws Exception { for(int numPCJs = 0; numPCJs <= 6; numPCJs++) { for(int pcjSPCount = 2; pcjSPCount <= 6; pcjSPCount++) { for(int querySPCount = 1; querySPCount <= 6; querySPCount++) { @@ -197,7 +197,7 @@ public class PCJOptimizerBenchmark { return parser.parseQuery(sparql, null).getTupleExpr(); } - private static PCJOptimizer makeUnchainedPCJOptimizer(final BenchmarkParams params) throws MalformedQueryException { + private static PCJOptimizer makeUnchainedPCJOptimizer(final BenchmarkParams params) throws Exception { final Queue<String> varQueue= Lists.newLinkedList(variables); final SPARQLParser parser = new SPARQLParser(); @@ -237,7 +237,7 @@ public class PCJOptimizerBenchmark { return new PCJOptimizer(indices, false); } - private static PCJOptimizer makeChainedPCJOptimizer(final BenchmarkParams params) throws MalformedQueryException { + private static PCJOptimizer makeChainedPCJOptimizer(final BenchmarkParams params) throws Exception { final Queue<String> varQueue= Lists.newLinkedList(variables); final SPARQLParser parser = new SPARQLParser();
