http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c826ffea/extras/indexing/src/test/java/org/apache/rya/indexing/external/PcjIntegrationTestingUtil.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/org/apache/rya/indexing/external/PcjIntegrationTestingUtil.java b/extras/indexing/src/test/java/org/apache/rya/indexing/external/PcjIntegrationTestingUtil.java index f8d841b..8b3b8f5 100644 --- a/extras/indexing/src/test/java/org/apache/rya/indexing/external/PcjIntegrationTestingUtil.java +++ b/extras/indexing/src/test/java/org/apache/rya/indexing/external/PcjIntegrationTestingUtil.java @@ -1,6 +1,4 @@ -package org.apache.rya.indexing.external; - -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,10 +16,12 @@ package org.apache.rya.indexing.external; * specific language governing permissions and limitations * under the License. */ +package org.apache.rya.indexing.external; import static com.google.common.base.Preconditions.checkNotNull; import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.Set; @@ -35,6 +35,12 @@ import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.data.Mutation; +import org.apache.rya.accumulo.AccumuloRdfConfiguration; +import org.apache.rya.api.model.VisibilityBindingSet; +import org.apache.rya.api.persist.RyaDAOException; +import org.apache.rya.indexing.accumulo.ConfigUtils; +import org.apache.rya.indexing.external.PrecomputedJoinIndexerConfig.PrecomputedJoinStorageType; +import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet; import org.apache.rya.indexing.pcj.storage.PcjException; import org.apache.rya.indexing.pcj.storage.PcjMetadata; import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjSerializer; @@ -43,6 +49,9 @@ import org.apache.rya.indexing.pcj.storage.accumulo.PcjTables; import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory; import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory; import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder; +import org.apache.rya.indexing.pcj.storage.mongo.MongoPcjDocuments; +import org.apache.rya.rdftriplestore.inference.InferenceEngineException; +import org.apache.rya.sail.config.RyaSailFactory; import org.openrdf.query.BindingSet; import org.openrdf.query.MalformedQueryException; import org.openrdf.query.QueryEvaluationException; @@ -62,14 +71,7 @@ import org.openrdf.sail.SailException; import com.google.common.base.Optional; import com.google.common.collect.Sets; - -import org.apache.rya.accumulo.AccumuloRdfConfiguration; -import org.apache.rya.api.persist.RyaDAOException; -import org.apache.rya.indexing.accumulo.ConfigUtils; -import org.apache.rya.indexing.external.PrecomputedJoinIndexerConfig.PrecomputedJoinStorageType; -import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet; -import org.apache.rya.rdftriplestore.inference.InferenceEngineException; -import org.apache.rya.sail.config.RyaSailFactory; +import com.mongodb.MongoClient; public class PcjIntegrationTestingUtil { @@ -96,7 +98,7 @@ public class PcjIntegrationTestingUtil { } } - public static SailRepository getPcjRepo(final String tablePrefix, final String instance) + public static SailRepository getAccumuloPcjRepo(final String tablePrefix, final String instance) throws AccumuloException, AccumuloSecurityException, RyaDAOException, RepositoryException, InferenceEngineException, NumberFormatException, UnknownHostException, SailException { @@ -104,26 +106,26 @@ public class PcjIntegrationTestingUtil { final AccumuloRdfConfiguration pcjConf = new AccumuloRdfConfiguration(); pcjConf.set(ConfigUtils.USE_PCJ, "true"); pcjConf.set(PrecomputedJoinIndexerConfig.PCJ_STORAGE_TYPE,PrecomputedJoinStorageType.ACCUMULO.name()); - populateTestConfig(instance, tablePrefix, pcjConf); + populateAccumuloConfig(instance, tablePrefix, pcjConf); final Sail pcjSail = RyaSailFactory.getInstance(pcjConf); final SailRepository pcjRepo = new SailRepository(pcjSail); return pcjRepo; } - public static SailRepository getNonPcjRepo(final String tablePrefix, + public static SailRepository getAccumuloNonPcjRepo(final String tablePrefix, final String instance) throws AccumuloException, - AccumuloSecurityException, RyaDAOException, RepositoryException, InferenceEngineException, - NumberFormatException, UnknownHostException, SailException { + AccumuloSecurityException, RyaDAOException, RepositoryException, InferenceEngineException, + NumberFormatException, UnknownHostException, SailException { final AccumuloRdfConfiguration nonPcjConf = new AccumuloRdfConfiguration(); - populateTestConfig(instance, tablePrefix, nonPcjConf); + populateAccumuloConfig(instance, tablePrefix, nonPcjConf); final Sail nonPcjSail = RyaSailFactory.getInstance(nonPcjConf); final SailRepository nonPcjRepo = new SailRepository(nonPcjSail); return nonPcjRepo; } - private static void populateTestConfig(final String instance, final String tablePrefix, final AccumuloRdfConfiguration config) { + private static void populateAccumuloConfig(final String instance, final String tablePrefix, final AccumuloRdfConfiguration config) { config.set(ConfigUtils.USE_MOCK_INSTANCE, "true"); config.set(ConfigUtils.CLOUDBASE_INSTANCE, instance); config.set(ConfigUtils.CLOUDBASE_USER, "test_user"); @@ -138,9 +140,13 @@ public class PcjIntegrationTestingUtil { repo.shutDown(); } + public static void deleteIndexDocuments(final MongoClient client, final String instance) { + client.getDatabase(instance).getCollection(MongoPcjDocuments.PCJ_COLLECTION_NAME).drop(); + } + public static void deleteIndexTables(final Connector accCon, final int tableNum, final String prefix) throws AccumuloException, AccumuloSecurityException, - TableNotFoundException { + TableNotFoundException { final TableOperations ops = accCon.tableOperations(); final String tablename = prefix + "INDEX_"; for (int i = 1; i < tableNum + 1; i++) { @@ -151,7 +157,7 @@ public class PcjIntegrationTestingUtil { } public static class BindingSetAssignmentCollector extends - QueryModelVisitorBase<RuntimeException> { + QueryModelVisitorBase<RuntimeException> { private final Set<QueryModelNode> bindingSetList = Sets.newHashSet(); @@ -172,7 +178,7 @@ public class PcjIntegrationTestingUtil { } public static class ExternalTupleVisitor extends - QueryModelVisitorBase<RuntimeException> { + QueryModelVisitorBase<RuntimeException> { private final Set<QueryModelNode> eSet = new HashSet<>(); @@ -195,13 +201,13 @@ public class PcjIntegrationTestingUtil { -//****************************Creation and Population of PcjTables*********************************** + //****************************Creation and Population of PcjTables Accumulo*************************** - /** + /** * Creates a new PCJ Table in Accumulo and populates it by scanning an * instance of Rya for historic matches. * <p> @@ -268,7 +274,7 @@ public class PcjIntegrationTestingUtil { */ public static void populatePcj(final Connector accumuloConn, final String pcjTableName, final RepositoryConnection ryaConn) - throws PcjException { + throws PcjException { checkNotNull(accumuloConn); checkNotNull(pcjTableName); checkNotNull(ryaConn); @@ -309,7 +315,7 @@ public class PcjIntegrationTestingUtil { public static void addResults(final Connector accumuloConn, final String pcjTableName, final Collection<BindingSet> results) - throws PcjException { + throws PcjException { checkNotNull(accumuloConn); checkNotNull(pcjTableName); checkNotNull(results); @@ -336,7 +342,7 @@ public class PcjIntegrationTestingUtil { */ private static void writeResults(final Connector accumuloConn, final String pcjTableName, final Collection<BindingSet> results) - throws PcjException { + throws PcjException { checkNotNull(accumuloConn); checkNotNull(pcjTableName); checkNotNull(results); @@ -387,7 +393,7 @@ public class PcjIntegrationTestingUtil { */ private static Set<Mutation> makeWriteResultMutations( final Set<VariableOrder> varOrders, final BindingSet result) - throws PcjException { + throws PcjException { checkNotNull(varOrders); checkNotNull(result); @@ -410,4 +416,112 @@ public class PcjIntegrationTestingUtil { return mutations; } + + //****************************Creation and Population of PcjTables Mongo *********************************** + + public static void deleteCoreRyaTables(final MongoClient client, final String instance, final String collName) { + final boolean bool = client.isLocked(); + client.getDatabase(instance).getCollection(collName).drop(); + } + + /** + * Creates a new PCJ Table in Accumulo and populates it by scanning an + * instance of Rya for historic matches. + * <p> + * If any portion of this operation fails along the way, the partially + * create PCJ table will be left in Accumulo. + * + * @param ryaConn - Connects to the Rya that will be scanned. (not null) + * @param mongoClient - Connects to the mongoDB that hosts the PCJ results. (not null) + * @param pcjName - The name of the PCJ table that will be created. (not null) + * @param sparql - The SPARQL query whose results will be loaded into the table. (not null) + * @throws PcjException The PCJ table could not be create or the values from Rya were + * not able to be loaded into it. + */ + public static void createAndPopulatePcj(final RepositoryConnection ryaConn, final MongoClient mongoClient, final String pcjName, final String instanceName, final String sparql) throws PcjException { + checkNotNull(ryaConn); + checkNotNull(mongoClient); + checkNotNull(pcjName); + checkNotNull(instanceName); + checkNotNull(sparql); + + final MongoPcjDocuments pcj = new MongoPcjDocuments(mongoClient, instanceName); + + pcj.createPcj(pcjName, sparql); + + // Load historic matches from Rya into the PCJ table. + populatePcj(pcj, pcjName, ryaConn); + } + + + /** + * Scan Rya for results that solve the PCJ's query and store them in the PCJ + * table. + * <p> + * This method assumes the PCJ table has already been created. + * + * @param mongoClient - A connection to the mongoDB that hosts the PCJ table. (not null) + * @param pcjTableName - The name of the PCJ table that will receive the results. (not null) + * @param ryaConn - A connection to the Rya store that will be queried to find results. (not null) + * @throws PcjException + * If results could not be written to the PCJ table, the PCJ + * table does not exist, or the query that is being execute was + * malformed. + */ + public static void populatePcj(final MongoPcjDocuments pcj, final String pcjTableName, final RepositoryConnection ryaConn) throws PcjException { + checkNotNull(pcj); + checkNotNull(pcjTableName); + checkNotNull(ryaConn); + + try { + // Fetch the query that needs to be executed from the PCJ table. + final PcjMetadata pcjMetadata = pcj.getPcjMetadata(pcjTableName); + final String sparql = pcjMetadata.getSparql(); + + // Query Rya for results to the SPARQL query. + final TupleQuery query = ryaConn.prepareTupleQuery(QueryLanguage.SPARQL, sparql); + final TupleQueryResult results = query.evaluate(); + + // Load batches of 1000 of them at a time into the PCJ table + final Set<BindingSet> batch = new HashSet<>(1000); + while (results.hasNext()) { + batch.add(results.next()); + + if (batch.size() == 1000) { + writeResults(pcj, pcjTableName, batch); + batch.clear(); + } + } + + if (!batch.isEmpty()) { + writeResults(pcj, pcjTableName, batch); + } + + } catch (RepositoryException | MalformedQueryException | QueryEvaluationException e) { + throw new PcjException("Could not populate a PCJ table with Rya results for the table named: " + pcjTableName, e); + } + } + + /** + * Add a collection of results to a specific PCJ table. + * + * @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null) + * @param pcjTableName - The name of the PCJ table that will receive the results. (not null) + * @param results - Binding sets that will be written to the PCJ table. (not null) + * @throws PcjException The provided PCJ table doesn't exist, is missing the + * PCJ metadata, or the result could not be written to it. + */ + private static void writeResults(final MongoPcjDocuments pcj, + final String pcjTableName, final Collection<BindingSet> results) + throws PcjException { + checkNotNull(pcj); + checkNotNull(pcjTableName); + checkNotNull(results); + + final Collection<VisibilityBindingSet> visRes = new ArrayList<>(); + results.forEach(bindingSet -> { + visRes.add(new VisibilityBindingSet(bindingSet)); + }); + pcj.addResults(pcjTableName, visRes); + } }
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c826ffea/extras/indexing/src/test/java/org/apache/rya/indexing/external/PrecompJoinOptimizerIT.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/org/apache/rya/indexing/external/PrecompJoinOptimizerIT.java b/extras/indexing/src/test/java/org/apache/rya/indexing/external/PrecompJoinOptimizerIT.java index c133a4a..86e5a8e 100644 --- a/extras/indexing/src/test/java/org/apache/rya/indexing/external/PrecompJoinOptimizerIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/indexing/external/PrecompJoinOptimizerIT.java @@ -74,10 +74,10 @@ public class PrecompJoinOptimizerIT { TableNotFoundException, InferenceEngineException, NumberFormatException, UnknownHostException, SailException { - repo = PcjIntegrationTestingUtil.getNonPcjRepo(tablePrefix, "instance"); + repo = PcjIntegrationTestingUtil.getAccumuloNonPcjRepo(tablePrefix, "instance"); conn = repo.getConnection(); - pcjRepo = PcjIntegrationTestingUtil.getPcjRepo(tablePrefix, "instance"); + pcjRepo = PcjIntegrationTestingUtil.getAccumuloPcjRepo(tablePrefix, "instance"); pcjConn = pcjRepo.getConnection(); sub = new URIImpl("uri:entity"); @@ -142,7 +142,7 @@ public class PrecompJoinOptimizerIT { final CountingResultHandler crh = new CountingResultHandler(); PcjIntegrationTestingUtil.deleteCoreRyaTables(accCon, tablePrefix); PcjIntegrationTestingUtil.closeAndShutdown(conn, repo); - repo = PcjIntegrationTestingUtil.getPcjRepo(tablePrefix, "instance"); + repo = PcjIntegrationTestingUtil.getAccumuloPcjRepo(tablePrefix, "instance"); conn = repo.getConnection(); conn.add(sub, talksTo, obj); conn.add(sub2, talksTo, obj2); @@ -231,7 +231,7 @@ public class PrecompJoinOptimizerIT { final CountingResultHandler crh = new CountingResultHandler(); PcjIntegrationTestingUtil.deleteCoreRyaTables(accCon, tablePrefix); PcjIntegrationTestingUtil.closeAndShutdown(conn, repo); - repo = PcjIntegrationTestingUtil.getPcjRepo(tablePrefix, "instance"); + repo = PcjIntegrationTestingUtil.getAccumuloPcjRepo(tablePrefix, "instance"); conn = repo.getConnection(); conn.add(sub, talksTo, obj); conn.add(sub2, talksTo, obj2); @@ -274,7 +274,7 @@ public class PrecompJoinOptimizerIT { final CountingResultHandler crh = new CountingResultHandler(); PcjIntegrationTestingUtil.deleteCoreRyaTables(accCon, tablePrefix); PcjIntegrationTestingUtil.closeAndShutdown(conn, repo); - repo = PcjIntegrationTestingUtil.getPcjRepo(tablePrefix, "instance"); + repo = PcjIntegrationTestingUtil.getAccumuloPcjRepo(tablePrefix, "instance"); conn = repo.getConnection(); conn.add(sub, talksTo, obj); conn.add(sub2, talksTo, obj2); @@ -321,7 +321,7 @@ public class PrecompJoinOptimizerIT { final CountingResultHandler crh = new CountingResultHandler(); PcjIntegrationTestingUtil.deleteCoreRyaTables(accCon, tablePrefix); PcjIntegrationTestingUtil.closeAndShutdown(conn, repo); - repo = PcjIntegrationTestingUtil.getPcjRepo(tablePrefix, "instance"); + repo = PcjIntegrationTestingUtil.getAccumuloPcjRepo(tablePrefix, "instance"); conn = repo.getConnection(); conn.add(sub, talksTo, obj); conn.add(sub, RDFS.LABEL, new LiteralImpl("label")); @@ -379,7 +379,7 @@ public class PrecompJoinOptimizerIT { PcjIntegrationTestingUtil.deleteCoreRyaTables(accCon, tablePrefix); PcjIntegrationTestingUtil.closeAndShutdown(conn, repo); - repo = PcjIntegrationTestingUtil.getPcjRepo(tablePrefix, "instance"); + repo = PcjIntegrationTestingUtil.getAccumuloPcjRepo(tablePrefix, "instance"); conn = repo.getConnection(); conn.add(sub2, RDF.TYPE, subclass2); conn.add(sub2, RDF.TYPE, obj2); @@ -445,7 +445,7 @@ public class PrecompJoinOptimizerIT { PcjIntegrationTestingUtil.deleteCoreRyaTables(accCon, tablePrefix); PcjIntegrationTestingUtil.closeAndShutdown(conn, repo); - repo = PcjIntegrationTestingUtil.getPcjRepo(tablePrefix, "instance"); + repo = PcjIntegrationTestingUtil.getAccumuloPcjRepo(tablePrefix, "instance"); conn = repo.getConnection(); conn.add(sub2, livesIn, city3); conn.add(sub, livesIn, city3); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c826ffea/extras/indexing/src/test/java/org/apache/rya/indexing/external/PrecompJoinOptimizerTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/org/apache/rya/indexing/external/PrecompJoinOptimizerTest.java b/extras/indexing/src/test/java/org/apache/rya/indexing/external/PrecompJoinOptimizerTest.java index 7c806f9..0035898 100644 --- a/extras/indexing/src/test/java/org/apache/rya/indexing/external/PrecompJoinOptimizerTest.java +++ b/extras/indexing/src/test/java/org/apache/rya/indexing/external/PrecompJoinOptimizerTest.java @@ -1,6 +1,4 @@ -package org.apache.rya.indexing.external; - -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,14 +16,28 @@ package org.apache.rya.indexing.external; * specific language governing permissions and limitations * under the License. */ +package org.apache.rya.indexing.external; + import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet; +import org.apache.rya.indexing.external.tupleSet.SimpleExternalTupleSet; +import org.apache.rya.indexing.mongodb.pcj.MongoPcjIndexSetProvider; +import org.apache.rya.indexing.pcj.matching.PCJOptimizer; +import org.apache.rya.indexing.pcj.matching.provider.AbstractPcjIndexSetProvider; +import org.apache.rya.indexing.pcj.matching.provider.AccumuloIndexSetProvider; +import org.apache.rya.mongodb.EmbeddedMongoSingleton; +import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration; import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.openrdf.query.algebra.Projection; import org.openrdf.query.algebra.QueryModelNode; import org.openrdf.query.algebra.StatementPattern; @@ -38,392 +50,406 @@ import org.openrdf.query.parser.sparql.SPARQLParser; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.apache.rya.indexing.external.tupleSet.ExternalTupleSet; -import org.apache.rya.indexing.external.tupleSet.SimpleExternalTupleSet; -import org.apache.rya.indexing.pcj.matching.PCJOptimizer; - +@RunWith(Parameterized.class) public class PrecompJoinOptimizerTest { + private final AbstractPcjIndexSetProvider provider; + + @Parameterized.Parameters + public static Collection providers() throws Exception { + final StatefulMongoDBRdfConfiguration conf = new StatefulMongoDBRdfConfiguration(new Configuration(), EmbeddedMongoSingleton.getNewMongoClient()); + return Lists.<AbstractPcjIndexSetProvider> newArrayList( + new AccumuloIndexSetProvider(new Configuration()), + new MongoPcjIndexSetProvider(conf) + ); + } + + public PrecompJoinOptimizerTest(final AbstractPcjIndexSetProvider provider) { + this.provider = provider; + } + @Test + public void testThreeIndex() throws Exception { + + final String q7 = ""// + + "SELECT ?s ?t ?u " // + + "{" // + + " ?s a ?t ."// + + " ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ."// + + " ?u <uri:talksTo> ?s . "// + + "}";// + + final String q8 = ""// + + "SELECT ?e ?l ?c " // + + "{" // + + " ?e a ?l ."// + + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?c ."// + + " ?c <uri:talksTo> ?e . "// + + "}";// + + final String q9 = ""// + + "SELECT ?f ?m ?d " // + + "{" // + + " ?f a ?m ."// + + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// + + " ?d <uri:talksTo> ?f . "// + + "}";// + + final String q15 = ""// + + "SELECT ?f ?m ?d ?e ?l ?c " // + + "{" // + + " ?f a ?m ."// + + " ?e a ?l ."// + + " ?d <uri:talksTo> ?f . "// + + " ?c <uri:talksTo> ?e . "// + + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// + + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?c ."// + + "}";// + + final SPARQLParser parser = new SPARQLParser(); + + final ParsedQuery pq1 = parser.parseQuery(q15, null); + final ParsedQuery pq2 = parser.parseQuery(q7, null); + final ParsedQuery pq3 = parser.parseQuery(q8, null); + final ParsedQuery pq4 = parser.parseQuery(q9, null); + + final SimpleExternalTupleSet extTup1 = new SimpleExternalTupleSet( + (Projection) pq2.getTupleExpr()); + final SimpleExternalTupleSet extTup2 = new SimpleExternalTupleSet( + (Projection) pq3.getTupleExpr()); + final SimpleExternalTupleSet extTup3 = new SimpleExternalTupleSet( + (Projection) pq4.getTupleExpr()); + + final List<ExternalTupleSet> list = new ArrayList<ExternalTupleSet>(); + + list.add(extTup1); + + final List<QueryModelNode> optTupNodes = Lists.newArrayList(); + optTupNodes.add(extTup2); + optTupNodes.add(extTup3); + + provider.setIndices(list); + final PCJOptimizer pcj = new PCJOptimizer(list, true, provider); + final TupleExpr te = pq1.getTupleExpr(); + pcj.optimize(te, null, null); + + final NodeCollector nc = new NodeCollector(); + te.visit(nc); + + Assert.assertEquals(nc.qNodes.size(), optTupNodes.size()); + for (final QueryModelNode node : nc.qNodes) { + Assert.assertTrue(optTupNodes.contains(node)); + } + + } + + @Test + public void testThreeIndex2() throws Exception { + + final String q1 = ""// + + "SELECT ?f ?m ?d ?e ?l ?c " // + + "{" // + + " ?f a ?m ."// + + " ?c a ?l ."// + + " ?d <uri:talksTo> ?f . "// + + " ?e <uri:talksTo> ?c . "// + + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// + + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?e ."// + + " ?m <uri:talksTo> ?e . "// + + "}";// + + final String q2 = ""// + + "SELECT ?u ?s ?t " // + + "{" // + + " ?s a ?t ."// + + " ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ."// + + " ?u <uri:talksTo> ?s . "// + + "}";// + + final String q3 = ""// + + "SELECT ?e ?c ?l " // + + "{" // + + " ?c a ?l ."// + + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?e ."// + + " ?e <uri:talksTo> ?c . "// + + "}";// + + final String q4 = ""// + + "SELECT ?d ?f ?m " // + + "{" // + + " ?f a ?m ."// + + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// + + " ?d <uri:talksTo> ?f . "// + + "}";// + + final SPARQLParser parser = new SPARQLParser(); + + final ParsedQuery pq1 = parser.parseQuery(q1, null); + final ParsedQuery pq2 = parser.parseQuery(q2, null); + final ParsedQuery pq3 = parser.parseQuery(q3, null); + final ParsedQuery pq4 = parser.parseQuery(q4, null); + + final SimpleExternalTupleSet extTup1 = new SimpleExternalTupleSet( + (Projection) pq2.getTupleExpr()); + final SimpleExternalTupleSet extTup2 = new SimpleExternalTupleSet( + (Projection) pq3.getTupleExpr()); + final SimpleExternalTupleSet extTup3 = new SimpleExternalTupleSet( + (Projection) pq4.getTupleExpr()); + + final List<ExternalTupleSet> list = new ArrayList<ExternalTupleSet>(); + + list.add(extTup1); + + final List<StatementPattern> spList = StatementPatternCollector + .process(pq1.getTupleExpr()); + final List<QueryModelNode> optTupNodes = Lists.newArrayList(); + optTupNodes.add(extTup3); + optTupNodes.add(spList.get(6)); + optTupNodes.add(extTup2); + + provider.setIndices(list); + final PCJOptimizer pcj = new PCJOptimizer(list, true, provider); + final TupleExpr te = pq1.getTupleExpr(); + pcj.optimize(te, null, null); + + final NodeCollector nc = new NodeCollector(); + te.visit(nc); + + Assert.assertEquals(nc.qNodes, Sets.newHashSet(optTupNodes)); + + } + + @Test + public void testSixIndex() throws Exception { + + final String q1 = ""// + + "SELECT ?f ?m ?d ?h ?i " // + + "{" // + + " ?f a ?m ."// + + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// + + " ?d <uri:talksTo> ?f . "// + + " ?d <uri:hangOutWith> ?f ." // + + " ?f <uri:hangOutWith> ?h ." // + + " ?f <uri:associatesWith> ?i ." // + + " ?i <uri:associatesWith> ?h ." // + + "}";// + + final String q2 = ""// + + "SELECT ?t ?s ?u " // + + "{" // + + " ?s a ?t ."// + + " ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ."// + + " ?u <uri:talksTo> ?s . "// + + "}";// + + final String q3 = ""// + + "SELECT ?s ?t ?u " // + + "{" // + + " ?s <uri:hangOutWith> ?t ." // + + " ?t <uri:hangOutWith> ?u ." // + + "}";// + + final String q4 = ""// + + "SELECT ?s ?t ?u " // + + "{" // + + " ?s <uri:associatesWith> ?t ." // + + " ?t <uri:associatesWith> ?u ." // + + "}";// + + final String q5 = ""// + + "SELECT ?m ?f ?d " // + + "{" // + + " ?f a ?m ."// + + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// + + " ?d <uri:talksTo> ?f . "// + + "}";// + + final String q6 = ""// + + "SELECT ?d ?f ?h " // + + "{" // + + " ?d <uri:hangOutWith> ?f ." // + + " ?f <uri:hangOutWith> ?h ." // + + "}";// + + final String q7 = ""// + + "SELECT ?f ?i ?h " // + + "{" // + + " ?f <uri:associatesWith> ?i ." // + + " ?i <uri:associatesWith> ?h ." // + + "}";// + + final SPARQLParser parser = new SPARQLParser(); + + final ParsedQuery pq1 = parser.parseQuery(q1, null); + final ParsedQuery pq2 = parser.parseQuery(q2, null); + final ParsedQuery pq3 = parser.parseQuery(q3, null); + final ParsedQuery pq4 = parser.parseQuery(q4, null); + final ParsedQuery pq5 = parser.parseQuery(q5, null); + final ParsedQuery pq6 = parser.parseQuery(q6, null); + final ParsedQuery pq7 = parser.parseQuery(q7, null); + + final SimpleExternalTupleSet extTup1 = new SimpleExternalTupleSet( + (Projection) pq2.getTupleExpr()); + final SimpleExternalTupleSet extTup2 = new SimpleExternalTupleSet( + (Projection) pq3.getTupleExpr()); + final SimpleExternalTupleSet extTup3 = new SimpleExternalTupleSet( + (Projection) pq4.getTupleExpr()); + final SimpleExternalTupleSet extTup4 = new SimpleExternalTupleSet( + (Projection) pq5.getTupleExpr()); + final SimpleExternalTupleSet extTup5 = new SimpleExternalTupleSet( + (Projection) pq6.getTupleExpr()); + final SimpleExternalTupleSet extTup6 = new SimpleExternalTupleSet( + (Projection) pq7.getTupleExpr()); + + final List<ExternalTupleSet> list = new ArrayList<ExternalTupleSet>(); + + list.add(extTup2); + list.add(extTup1); + list.add(extTup3); + + final List<QueryModelNode> optTupNodes = Lists.newArrayList(); + optTupNodes.add(extTup4); + optTupNodes.add(extTup6); + optTupNodes.add(extTup5); + + provider.setIndices(list); + final PCJOptimizer pcj = new PCJOptimizer(list, true, provider); + final TupleExpr te = pq1.getTupleExpr(); + pcj.optimize(te, null, null); + + System.out.println(te); + + final NodeCollector nc = new NodeCollector(); + te.visit(nc); + + Assert.assertEquals(nc.qNodes, Sets.newHashSet(optTupNodes)); + + } + + @Test + public void twoFourIndexWithFilterTest() throws Exception { + + final String q1 = ""// + + "SELECT ?f ?m ?d ?e ?l ?c " // + + "{" // + + " Filter(?f > \"5\")." // + + " Filter(?e > \"5\")." // + + " ?f a ?m ."// + + " ?e a ?l ."// + + " ?d <uri:talksTo> ?f . "// + + " ?c <uri:talksTo> ?e . "// + + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// + + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?c ."// + + "}";// + + final String q2 = ""// + + "SELECT ?s ?t ?u " // + + "{" // + + " ?s a ?t ."// + + " ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ."// + + " ?u <uri:talksTo> ?s . "// + + "}";// + + final String q3 = ""// + + "SELECT ?s ?t ?u " // + + "{" // + + " Filter(?s > \"5\") ."// + + " ?s a ?t ."// + + " ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ."// + + " ?u <uri:talksTo> ?s . "// + + "}";// + + final String q4 = ""// + + "SELECT ?f ?m ?d " // + + "{" // + + " Filter(?f > \"5\") ."// + + " ?f a ?m ."// + + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// + + " ?d <uri:talksTo> ?f . "// + + "}";// + + final String q5 = ""// + + "SELECT ?e ?l ?c " // + + "{" // + + " Filter(?e > \"5\") ."// + + " ?e a ?l ."// + + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?c ."// + + " ?c <uri:talksTo> ?e . "// + + "}";// + + final SPARQLParser parser = new SPARQLParser(); + + ParsedQuery pq1 = null; + ParsedQuery pq2 = null; + ParsedQuery pq3 = null; + ParsedQuery pq4 = null; + ParsedQuery pq5 = null; + + try { + pq1 = parser.parseQuery(q1, null); + pq2 = parser.parseQuery(q2, null); + pq3 = parser.parseQuery(q3, null); + pq4 = parser.parseQuery(q4, null); + pq5 = parser.parseQuery(q5, null); + + } catch (final Exception e) { + e.printStackTrace(); + } + + final SimpleExternalTupleSet extTup1 = new SimpleExternalTupleSet( + (Projection) pq2.getTupleExpr()); + final SimpleExternalTupleSet extTup2 = new SimpleExternalTupleSet( + (Projection) pq3.getTupleExpr()); + final SimpleExternalTupleSet extTup3 = new SimpleExternalTupleSet( + (Projection) pq4.getTupleExpr()); + final SimpleExternalTupleSet extTup4 = new SimpleExternalTupleSet( + (Projection) pq5.getTupleExpr()); - @Test - public void testThreeIndex() throws Exception { - - final String q7 = ""// - + "SELECT ?s ?t ?u " // - + "{" // - + " ?s a ?t ."// - + " ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ."// - + " ?u <uri:talksTo> ?s . "// - + "}";// - - final String q8 = ""// - + "SELECT ?e ?l ?c " // - + "{" // - + " ?e a ?l ."// - + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?c ."// - + " ?c <uri:talksTo> ?e . "// - + "}";// - - final String q9 = ""// - + "SELECT ?f ?m ?d " // - + "{" // - + " ?f a ?m ."// - + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// - + " ?d <uri:talksTo> ?f . "// - + "}";// - - final String q15 = ""// - + "SELECT ?f ?m ?d ?e ?l ?c " // - + "{" // - + " ?f a ?m ."// - + " ?e a ?l ."// - + " ?d <uri:talksTo> ?f . "// - + " ?c <uri:talksTo> ?e . "// - + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// - + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?c ."// - + "}";// - - final SPARQLParser parser = new SPARQLParser(); - - final ParsedQuery pq1 = parser.parseQuery(q15, null); - final ParsedQuery pq2 = parser.parseQuery(q7, null); - final ParsedQuery pq3 = parser.parseQuery(q8, null); - final ParsedQuery pq4 = parser.parseQuery(q9, null); - - final SimpleExternalTupleSet extTup1 = new SimpleExternalTupleSet( - (Projection) pq2.getTupleExpr()); - final SimpleExternalTupleSet extTup2 = new SimpleExternalTupleSet( - (Projection) pq3.getTupleExpr()); - final SimpleExternalTupleSet extTup3 = new SimpleExternalTupleSet( - (Projection) pq4.getTupleExpr()); - - final List<ExternalTupleSet> list = new ArrayList<ExternalTupleSet>(); - - list.add(extTup1); - - final List<QueryModelNode> optTupNodes = Lists.newArrayList(); - optTupNodes.add(extTup2); - optTupNodes.add(extTup3); - - final PCJOptimizer pcj = new PCJOptimizer(list, true); - final TupleExpr te = pq1.getTupleExpr(); - pcj.optimize(te, null, null); - - final NodeCollector nc = new NodeCollector(); - te.visit(nc); - - Assert.assertEquals(nc.qNodes.size(), optTupNodes.size()); - for (final QueryModelNode node : nc.qNodes) { - Assert.assertTrue(optTupNodes.contains(node)); - } - - } - - @Test - public void testThreeIndex2() throws Exception { - - final String q1 = ""// - + "SELECT ?f ?m ?d ?e ?l ?c " // - + "{" // - + " ?f a ?m ."// - + " ?c a ?l ."// - + " ?d <uri:talksTo> ?f . "// - + " ?e <uri:talksTo> ?c . "// - + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// - + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?e ."// - + " ?m <uri:talksTo> ?e . "// - + "}";// - - final String q2 = ""// - + "SELECT ?u ?s ?t " // - + "{" // - + " ?s a ?t ."// - + " ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ."// - + " ?u <uri:talksTo> ?s . "// - + "}";// - - final String q3 = ""// - + "SELECT ?e ?c ?l " // - + "{" // - + " ?c a ?l ."// - + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?e ."// - + " ?e <uri:talksTo> ?c . "// - + "}";// - - final String q4 = ""// - + "SELECT ?d ?f ?m " // - + "{" // - + " ?f a ?m ."// - + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// - + " ?d <uri:talksTo> ?f . "// - + "}";// - - final SPARQLParser parser = new SPARQLParser(); - - final ParsedQuery pq1 = parser.parseQuery(q1, null); - final ParsedQuery pq2 = parser.parseQuery(q2, null); - final ParsedQuery pq3 = parser.parseQuery(q3, null); - final ParsedQuery pq4 = parser.parseQuery(q4, null); - - final SimpleExternalTupleSet extTup1 = new SimpleExternalTupleSet( - (Projection) pq2.getTupleExpr()); - final SimpleExternalTupleSet extTup2 = new SimpleExternalTupleSet( - (Projection) pq3.getTupleExpr()); - final SimpleExternalTupleSet extTup3 = new SimpleExternalTupleSet( - (Projection) pq4.getTupleExpr()); - - final List<ExternalTupleSet> list = new ArrayList<ExternalTupleSet>(); - - list.add(extTup1); - - final List<StatementPattern> spList = StatementPatternCollector - .process(pq1.getTupleExpr()); - final List<QueryModelNode> optTupNodes = Lists.newArrayList(); - optTupNodes.add(extTup3); - optTupNodes.add(spList.get(6)); - optTupNodes.add(extTup2); - - final PCJOptimizer pcj = new PCJOptimizer(list, true); - final TupleExpr te = pq1.getTupleExpr(); - pcj.optimize(te, null, null); - - final NodeCollector nc = new NodeCollector(); - te.visit(nc); - - Assert.assertEquals(nc.qNodes, Sets.newHashSet(optTupNodes)); - - } - - @Test - public void testSixIndex() throws Exception { - - final String q1 = ""// - + "SELECT ?f ?m ?d ?h ?i " // - + "{" // - + " ?f a ?m ."// - + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// - + " ?d <uri:talksTo> ?f . "// - + " ?d <uri:hangOutWith> ?f ." // - + " ?f <uri:hangOutWith> ?h ." // - + " ?f <uri:associatesWith> ?i ." // - + " ?i <uri:associatesWith> ?h ." // - + "}";// - - final String q2 = ""// - + "SELECT ?t ?s ?u " // - + "{" // - + " ?s a ?t ."// - + " ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ."// - + " ?u <uri:talksTo> ?s . "// - + "}";// - - final String q3 = ""// - + "SELECT ?s ?t ?u " // - + "{" // - + " ?s <uri:hangOutWith> ?t ." // - + " ?t <uri:hangOutWith> ?u ." // - + "}";// - - final String q4 = ""// - + "SELECT ?s ?t ?u " // - + "{" // - + " ?s <uri:associatesWith> ?t ." // - + " ?t <uri:associatesWith> ?u ." // - + "}";// - - final String q5 = ""// - + "SELECT ?m ?f ?d " // - + "{" // - + " ?f a ?m ."// - + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// - + " ?d <uri:talksTo> ?f . "// - + "}";// - - final String q6 = ""// - + "SELECT ?d ?f ?h " // - + "{" // - + " ?d <uri:hangOutWith> ?f ." // - + " ?f <uri:hangOutWith> ?h ." // - + "}";// - - final String q7 = ""// - + "SELECT ?f ?i ?h " // - + "{" // - + " ?f <uri:associatesWith> ?i ." // - + " ?i <uri:associatesWith> ?h ." // - + "}";// - - final SPARQLParser parser = new SPARQLParser(); - - final ParsedQuery pq1 = parser.parseQuery(q1, null); - final ParsedQuery pq2 = parser.parseQuery(q2, null); - final ParsedQuery pq3 = parser.parseQuery(q3, null); - final ParsedQuery pq4 = parser.parseQuery(q4, null); - final ParsedQuery pq5 = parser.parseQuery(q5, null); - final ParsedQuery pq6 = parser.parseQuery(q6, null); - final ParsedQuery pq7 = parser.parseQuery(q7, null); - - final SimpleExternalTupleSet extTup1 = new SimpleExternalTupleSet( - (Projection) pq2.getTupleExpr()); - final SimpleExternalTupleSet extTup2 = new SimpleExternalTupleSet( - (Projection) pq3.getTupleExpr()); - final SimpleExternalTupleSet extTup3 = new SimpleExternalTupleSet( - (Projection) pq4.getTupleExpr()); - final SimpleExternalTupleSet extTup4 = new SimpleExternalTupleSet( - (Projection) pq5.getTupleExpr()); - final SimpleExternalTupleSet extTup5 = new SimpleExternalTupleSet( - (Projection) pq6.getTupleExpr()); - final SimpleExternalTupleSet extTup6 = new SimpleExternalTupleSet( - (Projection) pq7.getTupleExpr()); - - final List<ExternalTupleSet> list = new ArrayList<ExternalTupleSet>(); - - list.add(extTup2); - list.add(extTup1); - list.add(extTup3); - - final List<QueryModelNode> optTupNodes = Lists.newArrayList(); - optTupNodes.add(extTup4); - optTupNodes.add(extTup6); - optTupNodes.add(extTup5); - - final PCJOptimizer pcj = new PCJOptimizer(list, true); - final TupleExpr te = pq1.getTupleExpr(); - pcj.optimize(te, null, null); - - System.out.println(te); - - final NodeCollector nc = new NodeCollector(); - te.visit(nc); - - Assert.assertEquals(nc.qNodes, Sets.newHashSet(optTupNodes)); - - } - - @Test - public void twoFourIndexWithFilterTest() throws Exception { - - final String q1 = ""// - + "SELECT ?f ?m ?d ?e ?l ?c " // - + "{" // - + " Filter(?f > \"5\")." // - + " Filter(?e > \"5\")." // - + " ?f a ?m ."// - + " ?e a ?l ."// - + " ?d <uri:talksTo> ?f . "// - + " ?c <uri:talksTo> ?e . "// - + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// - + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?c ."// - + "}";// - - final String q2 = ""// - + "SELECT ?s ?t ?u " // - + "{" // - + " ?s a ?t ."// - + " ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ."// - + " ?u <uri:talksTo> ?s . "// - + "}";// - - final String q3 = ""// - + "SELECT ?s ?t ?u " // - + "{" // - + " Filter(?s > \"5\") ."// - + " ?s a ?t ."// - + " ?t <http://www.w3.org/2000/01/rdf-schema#label> ?u ."// - + " ?u <uri:talksTo> ?s . "// - + "}";// - - final String q4 = ""// - + "SELECT ?f ?m ?d " // - + "{" // - + " Filter(?f > \"5\") ."// - + " ?f a ?m ."// - + " ?m <http://www.w3.org/2000/01/rdf-schema#label> ?d ."// - + " ?d <uri:talksTo> ?f . "// - + "}";// - - final String q5 = ""// - + "SELECT ?e ?l ?c " // - + "{" // - + " Filter(?e > \"5\") ."// - + " ?e a ?l ."// - + " ?l <http://www.w3.org/2000/01/rdf-schema#label> ?c ."// - + " ?c <uri:talksTo> ?e . "// - + "}";// - - final SPARQLParser parser = new SPARQLParser(); - - ParsedQuery pq1 = null; - ParsedQuery pq2 = null; - ParsedQuery pq3 = null; - ParsedQuery pq4 = null; - ParsedQuery pq5 = null; - - try { - pq1 = parser.parseQuery(q1, null); - pq2 = parser.parseQuery(q2, null); - pq3 = parser.parseQuery(q3, null); - pq4 = parser.parseQuery(q4, null); - pq5 = parser.parseQuery(q5, null); - - } catch (final Exception e) { - e.printStackTrace(); - } - - final SimpleExternalTupleSet extTup1 = new SimpleExternalTupleSet( - (Projection) pq2.getTupleExpr()); - final SimpleExternalTupleSet extTup2 = new SimpleExternalTupleSet( - (Projection) pq3.getTupleExpr()); - final SimpleExternalTupleSet extTup3 = new SimpleExternalTupleSet( - (Projection) pq4.getTupleExpr()); - final SimpleExternalTupleSet extTup4 = new SimpleExternalTupleSet( - (Projection) pq5.getTupleExpr()); - - final List<ExternalTupleSet> list = new ArrayList<ExternalTupleSet>(); - - list.add(extTup2); - list.add(extTup1); - - final List<ExternalTupleSet> list2 = new ArrayList<ExternalTupleSet>(); - - list2.add(extTup3); - list2.add(extTup4); - - final PCJOptimizer pcj = new PCJOptimizer(list, true); - final TupleExpr te = pq1.getTupleExpr(); - pcj.optimize(te, null, null); - - System.out.println(te); - - final NodeCollector nc = new NodeCollector(); - te.visit(nc); - - Assert.assertEquals(nc.qNodes.size(), list2.size()); - - for (final QueryModelNode e : nc.qNodes) { - Assert.assertTrue(list2.contains(e)); - } - - } - - public static class NodeCollector extends - QueryModelVisitorBase<RuntimeException> { - - Set<QueryModelNode> qNodes = new HashSet<>(); - - @Override - public void meetNode(final QueryModelNode node) { - if (node instanceof StatementPattern - || node instanceof ExternalTupleSet) { - qNodes.add(node); - } - super.meetNode(node); - - } + final List<ExternalTupleSet> list = new ArrayList<ExternalTupleSet>(); + + list.add(extTup2); + list.add(extTup1); + + final List<ExternalTupleSet> list2 = new ArrayList<ExternalTupleSet>(); + + list2.add(extTup3); + list2.add(extTup4); + + provider.setIndices(list); + final PCJOptimizer pcj = new PCJOptimizer(list, true, provider); + final TupleExpr te = pq1.getTupleExpr(); + pcj.optimize(te, null, null); + + System.out.println(te); + + final NodeCollector nc = new NodeCollector(); + te.visit(nc); + + Assert.assertEquals(nc.qNodes.size(), list2.size()); + + for (final QueryModelNode e : nc.qNodes) { + Assert.assertTrue(list2.contains(e)); + } + + } + + public static class NodeCollector extends + QueryModelVisitorBase<RuntimeException> { + + Set<QueryModelNode> qNodes = new HashSet<>(); + + @Override + public void meetNode(final QueryModelNode node) { + if (node instanceof StatementPattern + || node instanceof ExternalTupleSet) { + qNodes.add(node); + } + super.meetNode(node); + + } public List<QueryModelNode> getNodes() { return Lists.newArrayList(qNodes); } - } + } }
