http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c12f58f4/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/VarConstantIndexListPruner.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/VarConstantIndexListPruner.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/VarConstantIndexListPruner.java index 7e72821..577663b 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/VarConstantIndexListPruner.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/VarConstantIndexListPruner.java @@ -8,9 +8,9 @@ package mvm.rya.indexing.IndexPlanValidator; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -20,6 +20,7 @@ package mvm.rya.indexing.IndexPlanValidator; */ +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -33,7 +34,7 @@ import org.openrdf.query.algebra.ValueConstant; import org.openrdf.query.algebra.Var; import org.openrdf.query.algebra.helpers.QueryModelVisitorBase; -import com.beust.jcommander.internal.Maps; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -54,9 +55,10 @@ public class VarConstantIndexListPruner implements IndexListPruner { queryFilterCount = cc.getFilterCount(); } - public Set<ExternalTupleSet> getRelevantIndices(List<ExternalTupleSet> indexList) { + @Override + public List<ExternalTupleSet> getRelevantIndices(List<ExternalTupleSet> indexList) { - Set<ExternalTupleSet> relIndexSet = Sets.newHashSet(); + List<ExternalTupleSet> relIndexSet = new ArrayList<>(); for (ExternalTupleSet e : indexList) { @@ -73,14 +75,14 @@ public class VarConstantIndexListPruner implements IndexListPruner { ConstantCollector cc = new ConstantCollector(); index.visit(cc); - + Map<String, Integer> indexConstantMap = cc.getConstantMap(); int indexSpCount = cc.getSpCount(); int indexFilterCount = cc.getFilterCount(); Set<String> indexConstants = indexConstantMap.keySet(); - if ((indexSpCount > querySpCount) || (indexFilterCount > queryFilterCount) - || !(Sets.intersection(indexConstants, queryConstantMap.keySet()).equals(indexConstants))) { + if (indexSpCount > querySpCount || indexFilterCount > queryFilterCount + || !Sets.intersection(indexConstants, queryConstantMap.keySet()).equals(indexConstants)) { return false; } @@ -99,31 +101,31 @@ public class VarConstantIndexListPruner implements IndexListPruner { private Map<String, Integer> constantMap = Maps.newHashMap(); private int spCount = 0; private int filterCount = 0; - - + + @Override public void meet(StatementPattern node) throws RuntimeException { - + spCount++; super.meet(node); } - - + + @Override public void meet(Filter node) throws RuntimeException { - + filterCount++; super.meet(node); } - - - - + + + + @Override public void meet(Var node) throws RuntimeException { - + if (node.isConstant()) { String key = node.getValue().toString(); if(constantMap.containsKey(key)){ @@ -136,12 +138,13 @@ public class VarConstantIndexListPruner implements IndexListPruner { } } - - - public void meet(ValueConstant node) throws RuntimeException { - + + + @Override + public void meet(ValueConstant node) throws RuntimeException { + String key = node.getValue().toString(); - + if(constantMap.containsKey(key)) { int count = constantMap.get(key); count += 1; @@ -149,23 +152,23 @@ public class VarConstantIndexListPruner implements IndexListPruner { } else { constantMap.put(key,1); } - + } - + public Map<String, Integer> getConstantMap() { return constantMap; } - + public int getSpCount(){ return spCount; } - + public int getFilterCount() { return filterCount; } - + } }
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c12f58f4/extras/indexing/src/main/java/mvm/rya/indexing/PcjQuery.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/PcjQuery.java b/extras/indexing/src/main/java/mvm/rya/indexing/PcjQuery.java new file mode 100644 index 0000000..57f77cf --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/PcjQuery.java @@ -0,0 +1,40 @@ +package mvm.rya.indexing; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import info.aduna.iteration.CloseableIteration; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import org.apache.accumulo.core.client.TableNotFoundException; +import org.openrdf.model.Value; +import org.openrdf.query.BindingSet; +import org.openrdf.query.QueryEvaluationException; + +public interface PcjQuery { + + public abstract CloseableIteration<BindingSet, QueryEvaluationException> queryPrecompJoin(List<String> varOrder, + String localityGroup, Map<String, Value> valMap, Map<String, String> varMap, Collection<BindingSet> constraints) throws QueryEvaluationException,TableNotFoundException; + +} + http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c12f58f4/extras/indexing/src/main/java/mvm/rya/indexing/PrecompQueryIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/PrecompQueryIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/PrecompQueryIndexer.java deleted file mode 100644 index 1aecd98..0000000 --- a/extras/indexing/src/main/java/mvm/rya/indexing/PrecompQueryIndexer.java +++ /dev/null @@ -1,63 +0,0 @@ -package mvm.rya.indexing; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -import info.aduna.iteration.CloseableIteration; - -import java.io.Closeable; -import java.io.Flushable; -import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import mvm.rya.indexing.external.tupleSet.AccumuloIndexSet.AccValueFactory; - -import org.apache.accumulo.core.client.TableNotFoundException; -import org.openrdf.model.Value; -import org.openrdf.query.BindingSet; -import org.openrdf.query.QueryEvaluationException; - - - -public interface PrecompQueryIndexer extends Closeable, Flushable { - - - public abstract void storeBindingSet(BindingSet bs) throws IOException ; - - public abstract void storeBindingSets(Collection<BindingSet> bindingSets) - throws IOException, IllegalArgumentException; - - - public abstract CloseableIteration<BindingSet, QueryEvaluationException> queryPrecompJoin(List<String> varOrder, - String localityGroup, Map<String, AccValueFactory> bindings, Map<String,Value> valMap, - Collection<BindingSet> constraints) throws QueryEvaluationException,TableNotFoundException; - - - - @Override - public abstract void flush() throws IOException; - - @Override - public abstract void close() throws IOException; -} - - http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c12f58f4/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityOptimizer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityOptimizer.java b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityOptimizer.java index e46c321..bb792ac 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityOptimizer.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityOptimizer.java @@ -8,9 +8,9 @@ package mvm.rya.indexing.accumulo.entity; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -27,7 +27,6 @@ import java.util.Set; import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.api.RdfCloudTripleStoreConfiguration; -import mvm.rya.api.persist.RdfEvalStatsDAO; import mvm.rya.api.persist.joinselect.SelectivityEvalDAO; import mvm.rya.indexing.accumulo.ConfigUtils; import mvm.rya.joinselect.AccumuloSelectivityEvalDAO; @@ -46,7 +45,6 @@ import org.openrdf.query.algebra.Join; import org.openrdf.query.algebra.QueryModelNode; import org.openrdf.query.algebra.StatementPattern; import org.openrdf.query.algebra.TupleExpr; -import org.openrdf.query.algebra.Var; import org.openrdf.query.algebra.evaluation.QueryOptimizer; import org.openrdf.query.algebra.helpers.QueryModelVisitorBase; @@ -60,12 +58,12 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { private RdfCloudTripleStoreConfiguration conf; private boolean isEvalDaoSet = false; - + public EntityOptimizer() { - + } - - public EntityOptimizer(RdfCloudTripleStoreConfiguration conf) { + + public EntityOptimizer(RdfCloudTripleStoreConfiguration conf) { if(conf.isUseStats() && conf.isUseSelectivity()) { try { eval = new AccumuloSelectivityEvalDAO(conf, ConfigUtils.getConnector(conf)); @@ -76,7 +74,7 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { } catch (AccumuloSecurityException e) { e.printStackTrace(); } - + isEvalDaoSet = true; } else { eval = null; @@ -84,13 +82,13 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { } this.conf = conf; } - + public EntityOptimizer(SelectivityEvalDAO<RdfCloudTripleStoreConfiguration> eval) { this.eval = eval; this.conf = eval.getConf(); isEvalDaoSet = true; } - + @Override public void setConf(Configuration conf) { if(conf instanceof RdfCloudTripleStoreConfiguration) { @@ -98,7 +96,7 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { } else { this.conf = new AccumuloRdfConfiguration(conf); } - + if (!isEvalDaoSet) { if(this.conf.isUseStats() && this.conf.isUseSelectivity()) { try { @@ -110,16 +108,16 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { } catch (AccumuloSecurityException e) { e.printStackTrace(); } - + isEvalDaoSet = true; } else { eval = null; isEvalDaoSet = true; } } - + } - + @Override public Configuration getConf() { return conf; @@ -151,7 +149,7 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { constructTuple(varMap, joinArgs, s); } List<TupleExpr> filterChain = getFilterChain(joinArgs); - + for (TupleExpr te : joinArgs) { if (!(te instanceof StatementPattern) || !(te instanceof EntityTupleSet)) { te.visit(this); @@ -164,12 +162,12 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { e.printStackTrace(); } } - + private List<TupleExpr> getFilterChain(List<TupleExpr> joinArgs) { List<TupleExpr> filterTopBottom = Lists.newArrayList(); TupleExpr filterChainTop = null; TupleExpr filterChainBottom = null; - + for(int i = 0; i < joinArgs.size(); i++) { if(joinArgs.get(i) instanceof Filter) { if(filterChainTop == null) { @@ -194,7 +192,7 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { } return filterTopBottom; } - + private TupleExpr getNewJoin(List<TupleExpr> joinArgs, List<TupleExpr> filterChain) { TupleExpr newJoin; @@ -278,7 +276,7 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { private void removeInvalidBins(HashMultimap<String, StatementPattern> varMap, boolean newMap) { Set<String> keys = Sets.newHashSet(varMap.keySet()); - + if (newMap) { for (String s : keys) { Set<StatementPattern> spSet = Sets.newHashSet(varMap.get(s)); @@ -333,7 +331,7 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { tempPriority = bin.size(); tempPriority *= getCardinality(bin); tempPriority *= getMinCardSp(bin); - + // weight starQuery where common Var is constant slightly more -- this factor is subject // to change if(s.startsWith("-const-")) { @@ -380,11 +378,11 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { double cardinality = Double.MAX_VALUE; double tempCard = -1; - + if(eval == null) { return 1; } - + List<StatementPattern> nodes = Lists.newArrayList(spNodes); AccumuloSelectivityEvalDAO ase = (AccumuloSelectivityEvalDAO) eval; @@ -418,7 +416,7 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { Join join = (Join) tupleExpr; getJoinArgs(join.getLeftArg(), joinArgs); getJoinArgs(join.getRightArg(), joinArgs); - } + } } else if(tupleExpr instanceof Filter) { joinArgs.add(tupleExpr); getJoinArgs(((Filter)tupleExpr).getArg(), joinArgs); @@ -431,6 +429,6 @@ public class EntityOptimizer implements QueryOptimizer, Configurable { } - + } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c12f58f4/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityTupleSet.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityTupleSet.java b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityTupleSet.java index dbe7a53..3944a59 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityTupleSet.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/EntityTupleSet.java @@ -8,9 +8,9 @@ package mvm.rya.indexing.accumulo.entity; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -24,7 +24,6 @@ import info.aduna.iteration.CloseableIteration; import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Set; @@ -56,7 +55,7 @@ import com.google.common.base.Joiner; public class EntityTupleSet extends ExternalSet implements ExternalBatchingIterator { - + private StarQuery starQuery; private RdfCloudTripleStoreConfiguration conf; private Set<String> variables; @@ -65,30 +64,30 @@ public class EntityTupleSet extends ExternalSet implements ExternalBatchingItera private double minCard; private Connector accCon = null; private boolean evalOptUsed = false; - + public EntityTupleSet() { - + } - + public EntityTupleSet(StarQuery sq, RdfCloudTripleStoreConfiguration conf) { this.starQuery = sq; this.conf = conf; - + variables = Sets.newHashSet(); if(!starQuery.commonVarConstant()) { variables.add(starQuery.getCommonVarName()); } variables.addAll(starQuery.getUnCommonVars()); - + init(); - + } - + public EntityTupleSet(StarQuery sq, RdfCloudTripleStoreConfiguration conf, boolean evalOptUsed) { this(sq,conf); this.evalOptUsed = evalOptUsed; } - + private void init() { try { @@ -129,11 +128,11 @@ public class EntityTupleSet extends ExternalSet implements ExternalBatchingItera public Set<String> getAssuredBindingNames() { return starQuery.getAssuredBindingNames(); } - + public Set<String> getVariables() { return variables; } - + @Override public String getSignature() { @@ -147,29 +146,29 @@ public class EntityTupleSet extends ExternalSet implements ExternalBatchingItera public void setStarQuery(StarQuery sq) { this.starQuery = sq; } - + @Override public EntityTupleSet clone() { StarQuery sq = new StarQuery(starQuery); return new EntityTupleSet(sq, conf); } - - + + @Override public double cardinality() { return cardinality; } - - + + public double getMinSpCard() { return minCard; } - - + + @Override public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(BindingSet bindings) throws QueryEvaluationException { - + // if starQuery contains node with cardinality less than 1000 and node // only has one variable, and number of SPs in starQuery is greater than 2, it is // more efficient to first evaluate this node and then pass the bindings @@ -180,7 +179,7 @@ public class EntityTupleSet extends ExternalSet implements ExternalBatchingItera RdfCloudTripleStoreConnection conn = getRyaSailConnection(); CloseableIteration<BindingSet, QueryEvaluationException> sol = (CloseableIteration<BindingSet, QueryEvaluationException>) conn .evaluate(minSp, null, bindings, false); - + Set<BindingSet> bSet = Sets.newHashSet(); while (sol.hasNext()) { //TODO this is not optimal - should check if bindings variables intersect minSp variables @@ -196,8 +195,8 @@ public class EntityTupleSet extends ExternalSet implements ExternalBatchingItera StarQuery sq = new StarQuery(spList); conn.close(); - - return (new EntityTupleSet(sq, conf, true)).evaluate(bSet); + + return new EntityTupleSet(sq, conf, true).evaluate(bSet); } catch (Exception e) { throw new QueryEvaluationException(e); @@ -206,27 +205,27 @@ public class EntityTupleSet extends ExternalSet implements ExternalBatchingItera this.evalOptUsed = true; return this.evaluate(Collections.singleton(bindings)); } - + } - - + + private int numberOfSpVars(StatementPattern sp) { List<Var> varList = sp.getVarList(); int varCount = 0; - + for(int i = 0; i < 3; i++) { if(!varList.get(i).isConstant()) { varCount++; } } - + return varCount; } - - + + @Override public CloseableIteration<BindingSet,QueryEvaluationException> evaluate(final Collection<BindingSet> bindingset) throws QueryEvaluationException { - + if(bindingset.size() < 2 && !this.evalOptUsed) { BindingSet bs = new QueryBindingSet(); if (bindingset.size() == 1) { @@ -245,8 +244,8 @@ public class EntityTupleSet extends ExternalSet implements ExternalBatchingItera IOUtils.closeQuietly(adi); } } - - + + private RdfCloudTripleStoreConnection getRyaSailConnection() throws AccumuloException, AccumuloSecurityException, SailException { final RdfCloudTripleStore store = new RdfCloudTripleStore(); @@ -259,6 +258,6 @@ public class EntityTupleSet extends ExternalSet implements ExternalBatchingItera return (RdfCloudTripleStoreConnection) store.getConnection(); } - - + + } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c12f58f4/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/StarQuery.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/StarQuery.java b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/StarQuery.java index e9d2f85..b40beb6 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/StarQuery.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/entity/StarQuery.java @@ -8,9 +8,9 @@ package mvm.rya.indexing.accumulo.entity; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -28,7 +28,6 @@ import java.util.Set; import mvm.rya.accumulo.documentIndex.TextColumn; import mvm.rya.api.domain.RyaType; import mvm.rya.api.domain.RyaURI; -import mvm.rya.api.persist.joinselect.SelectivityEvalDAO; import mvm.rya.api.resolver.RdfToRyaConversions; import mvm.rya.api.resolver.RyaContext; import mvm.rya.api.resolver.RyaTypeResolverException; @@ -47,7 +46,7 @@ import com.google.common.collect.Sets; import com.google.common.primitives.Bytes; public class StarQuery { - + private List<StatementPattern> nodes; private TextColumn[] nodeColumnCond; private String commonVarName; @@ -56,17 +55,17 @@ public class StarQuery { private String contextURI =""; private Map<String,Integer> varPos = Maps.newHashMap(); private boolean isCommonVarURI = false; - - + + public StarQuery(List<StatementPattern> nodes) { this.nodes = nodes; if(nodes.size() == 0) { throw new IllegalArgumentException("Nodes cannot be empty!"); } nodeColumnCond = new TextColumn[nodes.size()]; - Var tempContext = (Var) nodes.get(0).getContextVar(); + Var tempContext = nodes.get(0).getContextVar(); if(tempContext != null) { - context = (Var)tempContext.clone(); + context = tempContext.clone(); } else { context = new Var(); } @@ -76,51 +75,51 @@ public class StarQuery { e.printStackTrace(); } } - - + + public StarQuery(Set<StatementPattern> nodes) { this(Lists.newArrayList(nodes)); } - + public int size() { return nodes.size(); } - + public StarQuery(StarQuery other) { this(other.nodes); } - - + + public List<StatementPattern> getNodes() { return nodes; } - - + + public TextColumn[] getColumnCond() { return nodeColumnCond; } - - + + public boolean isCommonVarURI() { return isCommonVarURI; } - + public String getCommonVarName() { return commonVarName; } - + public Var getCommonVar() { return commonVar; } - + public boolean commonVarHasValue() { return commonVar.getValue() != null; } - + public boolean commonVarConstant() { return commonVar.isConstant(); } - + public String getCommonVarValue() { if(commonVarHasValue()) { return commonVar.getValue().stringValue(); @@ -128,75 +127,75 @@ public class StarQuery { return null; } } - - + + public Set<String> getUnCommonVars() { return varPos.keySet(); } - - + + public Map<String,Integer> getVarPos() { return varPos; } - + public boolean hasContext() { return context.getValue() != null; } - + public String getContextURI() { return contextURI; } - - - - + + + + public Set<String> getBindingNames() { - + Set<String> bindingNames = Sets.newHashSet(); - + for(StatementPattern sp: nodes) { - + if(bindingNames.size() == 0) { bindingNames = sp.getBindingNames(); } else { bindingNames = Sets.union(bindingNames, sp.getBindingNames()); } - + } - + return bindingNames; - + } - - - - + + + + public Set<String> getAssuredBindingNames() { - + Set<String> bindingNames = Sets.newHashSet(); - + for(StatementPattern sp: nodes) { - + if(bindingNames.size() == 0) { bindingNames = sp.getAssuredBindingNames(); } else { bindingNames = Sets.union(bindingNames, sp.getAssuredBindingNames()); } - + } - + return bindingNames; - + } - - - - - - - + + + + + + + public CardinalityStatementPattern getMinCardSp(AccumuloSelectivityEvalDAO ase) { - + StatementPattern minSp = null; double cardinality = Double.MAX_VALUE; double tempCard = -1; @@ -213,43 +212,43 @@ public class StarQuery { } catch (TableNotFoundException e) { e.printStackTrace(); } - + } return new CardinalityStatementPattern(minSp, cardinality) ; } - - - + + + public class CardinalityStatementPattern { - + private StatementPattern sp; private double cardinality; - + public CardinalityStatementPattern(StatementPattern sp, double cardinality) { this.sp = sp; this.cardinality = cardinality; } - + public StatementPattern getSp() { return sp; } - + public double getCardinality() { return cardinality; } - + } - - + + public double getCardinality( AccumuloSelectivityEvalDAO ase) { - + double cardinality = Double.MAX_VALUE; double tempCard = -1; ase.setDenormalized(true); - + try { for (int i = 0; i < nodes.size(); i++) { @@ -267,50 +266,50 @@ public class StarQuery { } catch (Exception e) { e.printStackTrace(); } - + ase.setDenormalized(false); return cardinality/(nodes.size() + 1); } - - - + + + public static Set<String> getCommonVars(StarQuery query, BindingSet bs) { - + Set<String> starQueryVarNames = Sets.newHashSet(); - + if(bs == null || bs.size() == 0) { return Sets.newHashSet(); } - + Set<String> bindingNames = bs.getBindingNames(); starQueryVarNames.addAll(query.getUnCommonVars()); if(!query.commonVarConstant()) { starQueryVarNames.add(query.getCommonVarName()); } - + return Sets.intersection(bindingNames, starQueryVarNames); - - + + } - - - - - - + + + + + + public static StarQuery getConstrainedStarQuery(StarQuery query, BindingSet bs) { - + if(bs.size() == 0) { return query; } - + Set<String> bindingNames = bs.getBindingNames(); Set<String> unCommonVarNames = query.getUnCommonVars(); Set<String> intersectVar = Sets.intersection(bindingNames, unCommonVarNames); - - + + if (!query.commonVarConstant()) { Value v = bs.getValue(query.getCommonVarName()); @@ -319,7 +318,7 @@ public class StarQuery { query.commonVar.setValue(v); } } - + for(String s: intersectVar) { try { query.nodeColumnCond[query.varPos.get(s)] = query.setValue(query.nodeColumnCond[query.varPos.get(s)], bs.getValue(s)); @@ -327,11 +326,11 @@ public class StarQuery { e.printStackTrace(); } } - + return query; } - - + + private TextColumn setValue(TextColumn tc, Value v) throws RyaTypeResolverException { String cq = tc.getColumnQualifier().toString(); @@ -355,22 +354,22 @@ public class StarQuery { return tc; } - - - + + + //assumes nodes forms valid star query with only one common variable //assumes nodes and commonVar has been set private TextColumn nodeToTextColumn(StatementPattern node, int i) throws RyaTypeResolverException { - + RyaContext rc = RyaContext.getInstance(); - + Var subjVar = node.getSubjectVar(); Var predVar = node.getPredicateVar(); Var objVar = node.getObjectVar(); - + RyaURI predURI = (RyaURI) RdfToRyaConversions.convertValue(node.getPredicateVar().getValue()); - - + + //assumes StatementPattern contains at least on variable if (subjVar.isConstant()) { if (commonVarConstant()) { @@ -415,46 +414,46 @@ public class StarQuery { return tc; } - - + + } - - + + } - - - - + + + + //called in constructor after nodes set //assumes nodes and nodeColumnCond are same size private void init() throws RyaTypeResolverException { - - + + commonVar = this.getCommonVar(nodes); if(!commonVar.isConstant()) { commonVarName = commonVar.getName(); } else { commonVarName = commonVar.getName().substring(7); } - + if(hasContext()) { RyaURI ctxtURI = (RyaURI) RdfToRyaConversions.convertValue(context.getValue()); contextURI = ctxtURI.getData(); } - + for(int i = 0; i < nodes.size(); i++){ nodeColumnCond[i] = nodeToTextColumn(nodes.get(i), i); } - + } - - - - - - - - + + + + + + + + // called after nodes set // assumes nodes forms valid query with single, common variable private Var getCommonVar(List<StatementPattern> nodes) { @@ -505,8 +504,8 @@ public class StarQuery { } } - - + + //assumes bindings is not of size 0 private static boolean isBindingsetValid(Set<String> bindings) { @@ -531,46 +530,46 @@ public class StarQuery { } } - - - - - + + + + + public static boolean isValidStarQuery(Collection<StatementPattern> nodes) { - + Set<String> bindings = null; boolean contextSet = false; Var context = null; - + if(nodes.size() < 2) { return false; } - + for(StatementPattern sp: nodes) { - + Var tempContext = sp.getContextVar(); Var predVar = sp.getPredicateVar(); - + //does not support variable context if(tempContext != null && !tempContext.isConstant()) { - return false; + return false; } if(!contextSet) { context = tempContext; contextSet = true; } else { - + if(context == null && tempContext != null) { return false; } else if (context != null && !context.equals(tempContext)) { return false; } } - + if(!predVar.isConstant()) { return false; } - + if(bindings == null ) { bindings = sp.getBindingNames(); if(bindings.size() == 0) { @@ -582,55 +581,56 @@ public class StarQuery { return false; } } - + } - - + + return isBindingsetValid(bindings); } - - - - - + + + + + // private static Set<String> getSpVariables(StatementPattern sp) { -// +// // Set<String> variables = Sets.newHashSet(); // List<Var> varList = sp.getVarList(); -// +// // for(Var v: varList) { // if(!v.isConstant()) { // variables.add(v.getName()); // } // } -// +// // return variables; -// +// // } -// - - - - - - public String toString() { - +// + + + + + + @Override + public String toString() { + String s = "Term conditions: " + "\n"; - - for(int i = 0; i < this.nodeColumnCond.length; i++) { - s = s + nodeColumnCond[i].toString() + "\n"; + + for (TextColumn element : this.nodeColumnCond) { + s = s + element.toString() + "\n"; } - - s = s + "Common Var: " + this.commonVar.toString() + "\n"; + + s = s + "Common Var: " + this.commonVar.toString() + "\n"; s = s + "Context: " + this.contextURI; - + return s; - + } - - - - - + + + + + } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c12f58f4/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/geo/GeoTupleSet.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/geo/GeoTupleSet.java b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/geo/GeoTupleSet.java index e7a5d68..2bc1bb0 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/geo/GeoTupleSet.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/accumulo/geo/GeoTupleSet.java @@ -8,9 +8,9 @@ package mvm.rya.indexing.accumulo.geo; * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -31,14 +31,12 @@ import mvm.rya.indexing.IteratorFactory; import mvm.rya.indexing.SearchFunction; import mvm.rya.indexing.StatementContraints; import mvm.rya.indexing.external.tupleSet.ExternalTupleSet; -import mvm.rya.indexing.external.tupleSet.SimpleExternalTupleSet; import org.apache.hadoop.conf.Configuration; import org.openrdf.model.Statement; import org.openrdf.model.URI; import org.openrdf.query.BindingSet; import org.openrdf.query.QueryEvaluationException; -import org.openrdf.query.algebra.QueryModelVisitor; import com.google.common.base.Joiner; import com.google.common.collect.Maps; @@ -46,14 +44,14 @@ import com.vividsolutions.jts.geom.Geometry; import com.vividsolutions.jts.io.ParseException; import com.vividsolutions.jts.io.WKTReader; -//Indexing Node for geo expressions to be inserted into execution plan +//Indexing Node for geo expressions to be inserted into execution plan //to delegate geo portion of query to geo index public class GeoTupleSet extends ExternalTupleSet { private Configuration conf; private GeoIndexer geoIndexer; private IndexingExpr filterInfo; - + public GeoTupleSet(IndexingExpr filterInfo, GeoIndexer geoIndexer) { this.filterInfo = filterInfo; @@ -66,7 +64,8 @@ public class GeoTupleSet extends ExternalTupleSet { return filterInfo.getBindingNames(); } - public GeoTupleSet clone() { + @Override + public GeoTupleSet clone() { return new GeoTupleSet(filterInfo, geoIndexer); } @@ -74,15 +73,15 @@ public class GeoTupleSet extends ExternalTupleSet { public double cardinality() { return 0.0; // No idea how the estimate cardinality here. } - - + + @Override public String getSignature() { return "(GeoTuple Projection) " + "variables: " + Joiner.on(", ").join(this.getBindingNames()).replaceAll("\\s+", " "); } - - - + + + @Override public boolean equals(Object other) { if (other == this) { @@ -94,16 +93,16 @@ public class GeoTupleSet extends ExternalTupleSet { GeoTupleSet arg = (GeoTupleSet) other; return this.filterInfo.equals(arg.filterInfo); } - + @Override public int hashCode() { int result = 17; result = 31*result + filterInfo.hashCode(); - + return result; } - - + + /** * Returns an iterator over the result set of the contained IndexingExpr. @@ -114,37 +113,37 @@ public class GeoTupleSet extends ExternalTupleSet { @Override public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(BindingSet bindings) throws QueryEvaluationException { - - + + URI funcURI = filterInfo.getFunction(); - SearchFunction searchFunction = (new GeoSearchFunctionFactory(conf)).getSearchFunction(funcURI); + SearchFunction searchFunction = new GeoSearchFunctionFactory(conf).getSearchFunction(funcURI); if(filterInfo.getArguments().length > 1) { throw new IllegalArgumentException("Index functions do not support more than two arguments."); } - + String queryText = filterInfo.getArguments()[0].stringValue(); - + return IteratorFactory.getIterator(filterInfo.getSpConstraint(), bindings, queryText, searchFunction); } - + //returns appropriate search function for a given URI //search functions used in GeoMesaGeoIndexer to access index public class GeoSearchFunctionFactory { - + Configuration conf; - + private final Map<URI, SearchFunction> SEARCH_FUNCTION_MAP = Maps.newHashMap(); public GeoSearchFunctionFactory(Configuration conf) { this.conf = conf; } - + /** * Get a {@link GeoSearchFunction} for a given URI. - * + * * @param searchFunction * @return */ @@ -359,6 +358,6 @@ public class GeoTupleSet extends ExternalTupleSet { } } - + } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c12f58f4/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalIndexMain.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalIndexMain.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalIndexMain.java deleted file mode 100644 index c4e55be..0000000 --- a/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalIndexMain.java +++ /dev/null @@ -1,219 +0,0 @@ -package mvm.rya.indexing.external; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import mvm.rya.accumulo.AccumuloRdfConfiguration; -import mvm.rya.accumulo.AccumuloRyaDAO; -import mvm.rya.indexing.accumulo.ConfigUtils; -import mvm.rya.indexing.external.tupleSet.AccumuloIndexSet; -import mvm.rya.indexing.external.tupleSet.ExternalTupleSet; -import mvm.rya.rdftriplestore.RdfCloudTripleStore; - -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.client.ZooKeeperInstance; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.openrdf.query.BindingSet; -import org.openrdf.query.QueryLanguage; -import org.openrdf.query.QueryResultHandlerException; -import org.openrdf.query.TupleQueryResultHandler; -import org.openrdf.query.TupleQueryResultHandlerException; -import org.openrdf.repository.sail.SailRepository; -import org.openrdf.repository.sail.SailRepositoryConnection; -import org.openrdf.sail.Sail; - -import com.beust.jcommander.internal.Maps; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -public class ExternalIndexMain { - - private static String userStr = ""; - private static String passStr = ""; - - private static String instStr = ""; - private static String zooStr = ""; - - private static String tablePrefix = ""; - - private static String AUTHS = ""; - - public static void main(String[] args) throws Exception { - Preconditions.checkArgument(args.length == 6, "java " + ExternalIndexMain.class.getCanonicalName() - + " sparqlFile cbinstance cbzk cbuser cbpassword rdfTablePrefix."); - - final String sparqlFile = args[0]; - - instStr = args[1]; - zooStr = args[2]; - userStr = args[3]; - passStr = args[4]; - tablePrefix = args[5]; - - String queryString = FileUtils.readFileToString(new File(sparqlFile)); - - - // Look for Extra Indexes - Instance inst = new ZooKeeperInstance(instStr, zooStr); - Connector c = inst.getConnector(userStr, passStr.getBytes()); - - System.out.println("Searching for Indexes"); - Map<String, String> indexTables = Maps.newLinkedHashMap(); - for (String table : c.tableOperations().list()) { - if (table.startsWith(tablePrefix + "INDEX_")) { - Scanner s = c.createScanner(table, new Authorizations()); - s.setRange(Range.exact(new Text("~SPARQL"))); - for (Entry<Key, Value> e : s) { - indexTables.put(table, e.getValue().toString()); - } - } - } - - List<ExternalTupleSet> index = Lists.newArrayList(); - - if (indexTables.isEmpty()) { - System.out.println("No Index found"); - } else { - for (String table : indexTables.keySet()) { - String indexSparqlString = indexTables.get(table); - System.out.println("====================== INDEX FOUND ======================"); - System.out.println(" table : " + table); - System.out.println(" sparql : "); - System.out.println(indexSparqlString); - - index.add(new AccumuloIndexSet(indexSparqlString, c, table)); - } - } - - // Connect to Rya - Sail s = getRyaSail(); - SailRepository repo = new SailRepository(s); - repo.initialize(); - - // Perform Query - - CountingTupleQueryResultHandler count = new CountingTupleQueryResultHandler(); - - SailRepositoryConnection conn; - if (index.isEmpty()) { - conn = repo.getConnection(); - - } else { - ExternalProcessor processor = new ExternalProcessor(index); - - Sail processingSail = new ExternalSail(s, processor); - SailRepository smartSailRepo = new SailRepository(processingSail); - smartSailRepo.initialize(); - - conn = smartSailRepo.getConnection(); - } - - startTime = System.currentTimeMillis(); - lastTime = startTime; - System.out.println("Query Started"); - conn.prepareTupleQuery(QueryLanguage.SPARQL, queryString).evaluate(count); - - System.out.println("Count of Results found : " + count.i); - System.out.println("Total query time (s) : " + (System.currentTimeMillis() - startTime) / 1000.); - } - - static long lastTime = 0; - static long startTime = 0; - - private static class CountingTupleQueryResultHandler implements TupleQueryResultHandler { - public int i = 0; - - @Override - public void handleBoolean(boolean value) throws QueryResultHandlerException { - } - - @Override - public void handleLinks(List<String> linkUrls) throws QueryResultHandlerException { - } - - @Override - public void startQueryResult(List<String> bindingNames) throws TupleQueryResultHandlerException { - System.out.println("First Result Recieved (s) : " + (System.currentTimeMillis() - startTime) / 1000.); - } - - @Override - public void endQueryResult() throws TupleQueryResultHandlerException { - } - - @Override - public void handleSolution(BindingSet bindingSet) throws TupleQueryResultHandlerException { - i++; - if (i % 10 == 0) { - long mark = System.currentTimeMillis(); - System.out.println("Count : " + i + ". Time (s) : " + (mark - lastTime) / 1000.); - lastTime = mark; - } - - } - - } - - private static Configuration getConf() { - - Configuration conf = new Configuration(); - - conf.set(ConfigUtils.CLOUDBASE_USER, userStr); - conf.set(ConfigUtils.CLOUDBASE_PASSWORD, passStr); - - conf.set(ConfigUtils.CLOUDBASE_INSTANCE, instStr); - conf.set(ConfigUtils.CLOUDBASE_ZOOKEEPERS, zooStr); - - conf.set(ConfigUtils.CLOUDBASE_AUTHS, AUTHS); - conf.setBoolean(ConfigUtils.DISPLAY_QUERY_PLAN, true); - return conf; - } - - private static Sail getRyaSail() throws AccumuloException, AccumuloSecurityException { - - Connector connector = ConfigUtils.getConnector(getConf()); - - final RdfCloudTripleStore store = new RdfCloudTripleStore(); - AccumuloRyaDAO crdfdao = new AccumuloRyaDAO(); - crdfdao.setConnector(connector); - - AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(getConf()); - conf.setTablePrefix(tablePrefix); - crdfdao.setConf(conf); - store.setRyaDAO(crdfdao); - - return store; - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c12f58f4/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalProcessor.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalProcessor.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalProcessor.java deleted file mode 100644 index 2c6d924..0000000 --- a/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalProcessor.java +++ /dev/null @@ -1,726 +0,0 @@ -package mvm.rya.indexing.external; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import mvm.rya.indexing.external.QueryVariableNormalizer.VarCollector; -import mvm.rya.indexing.external.tupleSet.ExternalTupleSet; - -import org.openrdf.query.algebra.BindingSetAssignment; -import org.openrdf.query.algebra.Filter; -import org.openrdf.query.algebra.Join; -import org.openrdf.query.algebra.Projection; -import org.openrdf.query.algebra.QueryModelNode; -import org.openrdf.query.algebra.StatementPattern; -import org.openrdf.query.algebra.TupleExpr; -import org.openrdf.query.algebra.Var; -import org.openrdf.query.algebra.helpers.QueryModelVisitorBase; -import org.openrdf.query.algebra.helpers.StatementPatternCollector; - -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -/** - * Processes a {@link TupleExpr} and replaces sets of elements in the tree with {@link ExternalTupleSet} objects. - */ -public class ExternalProcessor { - - private List<ExternalTupleSet> indexSet; - - public ExternalProcessor(List<ExternalTupleSet> indexSet) { - this.indexSet = indexSet; - } - - /** - * Iterates through list of indexes and replaces all subtrees of query which match index with external tuple object built from index. - * - * @param query - * @return TupleExpr - */ - public TupleExpr process(TupleExpr query) { - TupleExpr rtn = query.clone(); - - - //move BindingSetAssignment Nodes out of the way - organizeBSAs(rtn); - - - // test to see if query contains no other nodes - // than filter, join, projection, and statement pattern and - // test whether query contains duplicate StatementPatterns and filters - if (isTupleValid(rtn)) { - - for (ExternalTupleSet index : indexSet) { - - // test to see if index contains at least one StatementPattern, - // that StatementPatterns are unique, - // and that all variables found in filters occur in some - // StatementPattern - if (isTupleValid(index.getTupleExpr())) { - - List<TupleExpr> normalize = getMatches(rtn, index.getTupleExpr()); - - for (TupleExpr tup : normalize) { - ExternalTupleSet eTup = (ExternalTupleSet) index.clone(); - setTableMap(tup, eTup); - setSupportedVarOrderMap(eTup); - eTup.setProjectionExpr((Projection) tup); - SPBubbleDownVisitor indexVistor = new SPBubbleDownVisitor(eTup); - rtn.visit(indexVistor); - FilterBubbleManager fbmv = new FilterBubbleManager(eTup); - rtn.visit(fbmv); - SubsetEqualsVisitor subIndexVis = new SubsetEqualsVisitor(eTup); - rtn.visit(subIndexVis); - - } - } - - } - - return rtn; - } else { - throw new IllegalArgumentException("Invalid Query."); - } - } - - - private void setTableMap(TupleExpr tupleMatch, ExternalTupleSet index) { - - List<String> replacementVars = Lists.newArrayList(tupleMatch.getBindingNames()); - List<String> tableVars = Lists.newArrayList(index.getTupleExpr().getBindingNames()); - - Map<String,String> tableMap = Maps.newHashMap(); - - for(int i = 0; i < tableVars.size(); i++) { - tableMap.put(replacementVars.get(i), tableVars.get(i)); - } - index.setTableVarMap(tableMap); - - - } - - private void setSupportedVarOrderMap(ExternalTupleSet index) { - - Map<String, Set<String>> supportedVarOrders = Maps.newHashMap(); - BiMap<String, String> biMap = HashBiMap.create(index.getTableVarMap()).inverse(); - Map<String, Set<String>> oldSupportedVarOrders = index.getSupportedVariableOrderMap(); - - Set<String> temp = null; - Set<String> keys = oldSupportedVarOrders.keySet(); - - for (String s : keys) { - temp = oldSupportedVarOrders.get(s); - Set<String> newSet = Sets.newHashSet(); - - for (String t : temp) { - newSet.add(biMap.get(t)); - } - - String[] tempStrings = s.split("\u0000"); - String v = ""; - for(String u: tempStrings) { - if(v.length() == 0){ - v = v + biMap.get(u); - } else { - v = v + "\u0000" + biMap.get(u); - } - } - - supportedVarOrders.put(v, newSet); - - } - - index.setSupportedVariableOrderMap(supportedVarOrders); - - } - - - - private List<TupleExpr> getMatches(TupleExpr query, TupleExpr tuple) { - - try { - List<TupleExpr> list = QueryVariableNormalizer.getNormalizedIndex(query, tuple); - // System.out.println("Match list is " + list); - return list; - } catch (Exception e) { - System.out.println(e); - } - - return new ArrayList<TupleExpr>(); - - } - - // determines whether query is valid, which requires that a - // query must contain a StatementPattern, not contain duplicate - // Statement Patterns or Filters, not be comprised of only Projection, - // Join, StatementPattern, and Filter nodes, and that any variable - // appearing in a Filter must appear in a StatementPattern. - private static boolean isTupleValid(QueryModelNode node) { - - ValidQueryVisitor vqv = new ValidQueryVisitor(); - node.visit(vqv); - - Set<String> spVars = getVarNames(getQNodes("sp", node)); - - if (vqv.isValid() && (spVars.size() > 0)) { - - FilterCollector fvis = new FilterCollector(); - node.visit(fvis); - List<QueryModelNode> fList = fvis.getFilters(); - return (fList.size() == Sets.newHashSet(fList).size() && getVarNames(fList).size() <= spVars.size()); - - } else { - return false; - } - } - - private static Set<QueryModelNode> getQNodes(QueryModelNode queryNode) { - Set<QueryModelNode> rtns = new HashSet<QueryModelNode>(); - - StatementPatternCollector spc = new StatementPatternCollector(); - queryNode.visit(spc); - rtns.addAll(spc.getStatementPatterns()); - - FilterCollector fvis = new FilterCollector(); - queryNode.visit(fvis); - rtns.addAll(fvis.getFilters()); - - ExternalTupleCollector eVis = new ExternalTupleCollector(); - queryNode.visit(eVis); - rtns.addAll(eVis.getExtTup()); - - return rtns; - } - - private static Set<QueryModelNode> getQNodes(String node, QueryModelNode queryNode) { - - if (node.equals("sp")) { - Set<QueryModelNode> eSet = new HashSet<QueryModelNode>(); - StatementPatternCollector spc = new StatementPatternCollector(); - queryNode.visit(spc); - List<StatementPattern> spList = spc.getStatementPatterns(); - eSet.addAll(spList); - // returns empty set if list contains duplicate StatementPatterns - if (spList.size() > eSet.size()) { - return Sets.newHashSet(); - } else { - return eSet; - } - } else if (node.equals("filter")) { - - FilterCollector fvis = new FilterCollector(); - queryNode.visit(fvis); - - return Sets.newHashSet(fvis.getFilters()); - } else { - - throw new IllegalArgumentException("Invalid node type."); - } - } - - // moves StatementPatterns in query that also occur in index to bottom of - // query tree. - private static class SPBubbleDownVisitor extends QueryModelVisitorBase<RuntimeException> { - - private TupleExpr tuple; - private QueryModelNode indexQNode; - private Set<QueryModelNode> sSet = Sets.newHashSet(); - - public SPBubbleDownVisitor(ExternalTupleSet index) { - - this.tuple = index.getTupleExpr(); - indexQNode = ((Projection) tuple).getArg(); - sSet = getQNodes("sp", indexQNode); - - } - - public void meet(Projection node) { - // moves external tuples above statement patterns before attempting - // to bubble down index statement patterns found in query tree - - organizeExtTuples(node); - - super.meet(node); - } - - public void meet(Join node) { - // if right node contained in index, move it to bottom of query tree - if (sSet.contains(node.getRightArg())) { - - Set<QueryModelNode> eSet = getQNodes("sp", node); - Set<QueryModelNode> compSet = Sets.difference(eSet, sSet); - - if (eSet.containsAll(sSet)) { - - QNodeExchanger qne = new QNodeExchanger(node.getRightArg(), compSet); - node.visit(qne); - node.replaceChildNode(node.getRightArg(), qne.getReplaced()); - - super.meet(node); - } - return; - } - // if left node contained in index, move it to bottom of query tree - else if (sSet.contains(node.getLeftArg())) { - - Set<QueryModelNode> eSet = getQNodes("sp", node); - Set<QueryModelNode> compSet = Sets.difference(eSet, sSet); - - if (eSet.containsAll(sSet)) { - - QNodeExchanger qne = new QNodeExchanger(node.getLeftArg(), compSet); - node.visit(qne); - node.replaceChildNode(node.getLeftArg(), qne.getReplaced()); - - super.meet(node); - } - return; - - } else { - super.meet(node); - } - - } - - // moves all ExternalTupleSets in query tree above remaining - // StatementPatterns - private static void organizeExtTuples(QueryModelNode node) { - - ExternalTupleCollector eVis = new ExternalTupleCollector(); - node.visit(eVis); - - ExtTupleExchangeVisitor oev = new ExtTupleExchangeVisitor(eVis.getExtTup()); - node.visit(oev); - } - - } - - // given a replacement QueryModelNode and compSet, this visitor replaces the - // first - // element in the query tree that occurs in compSet with replacement and - // returns - // the element that was replaced. - private static class QNodeExchanger extends QueryModelVisitorBase<RuntimeException> { - - private QueryModelNode toBeReplaced; - private QueryModelNode replacement; - private Set<QueryModelNode> compSet; - - public QNodeExchanger(QueryModelNode replacement, Set<QueryModelNode> compSet) { - this.replacement = replacement; - this.toBeReplaced = replacement; - this.compSet = compSet; - } - - public QueryModelNode getReplaced() { - return toBeReplaced; - } - - public void meet(Join node) { - - if (compSet.contains(node.getRightArg())) { - this.toBeReplaced = node.getRightArg(); - node.replaceChildNode(node.getRightArg(), replacement); - return; - } else if (compSet.contains(node.getLeftArg())) { - this.toBeReplaced = node.getLeftArg(); - node.replaceChildNode(node.getLeftArg(), replacement); - return; - } else { - super.meet(node); - } - - } - - } - - // moves filter that occurs in both query and index down the query tree so - // that that it is positioned - // above statement patterns associated with index. Precondition for calling - // this method is that - // SPBubbleDownVisitor has been called to position index StatementPatterns - // within query tree. - //TODO this visitor assumes that all filters are positioned at top of query tree - //could lead to problems if filter optimizer called before external processor - private static class FilterBubbleDownVisitor extends QueryModelVisitorBase<RuntimeException> { - - private QueryModelNode filter; - private Set<QueryModelNode> compSet; - private boolean filterPlaced = false; - - public FilterBubbleDownVisitor(QueryModelNode filter, Set<QueryModelNode> compSet) { - this.filter = filter; - this.compSet = compSet; - - } - - public boolean filterPlaced() { - return filterPlaced; - } - - public void meet(Join node) { - - if (!compSet.contains(node.getRightArg())) { - // looks for placed to position filter node. if right node is - // contained in index - // and left node is statement pattern node contained in index or - // is a join, place - // filter above join. - if (node.getLeftArg() instanceof Join || !(compSet.contains(node.getLeftArg()))) { - - QueryModelNode pNode = node.getParentNode(); - ((Filter) filter).setArg(node); - pNode.replaceChildNode(node, filter); - filterPlaced = true; - - return; - } // otherwise place filter below join and above right arg - else { - ((Filter) filter).setArg(node.getRightArg()); - node.replaceChildNode(node.getRightArg(), filter); - filterPlaced = true; - return; - - } - } else if ((node.getLeftArg() instanceof StatementPattern) && !compSet.contains(node.getLeftArg())) { - - ((Filter) filter).setArg(node.getLeftArg()); - node.replaceChildNode(node.getLeftArg(), filter); - filterPlaced = true; - - return; - } else { - super.meet(node); - } - } - - } - - private static Set<String> getVarNames(Collection<QueryModelNode> nodes) { - - List<String> tempVars; - Set<String> nodeVarNames = Sets.newHashSet(); - - for (QueryModelNode s : nodes) { - tempVars = VarCollector.process(s); - for (String t : tempVars) - nodeVarNames.add(t); - } - return nodeVarNames; - - } - - // visitor which determines whether or not to reposition a filter by calling - // FilterBubbleDownVisitor - private static class FilterBubbleManager extends QueryModelVisitorBase<RuntimeException> { - - private TupleExpr tuple; - private QueryModelNode indexQNode; - private Set<QueryModelNode> sSet = Sets.newHashSet(); - private Set<QueryModelNode> bubbledFilters = Sets.newHashSet(); - - public FilterBubbleManager(ExternalTupleSet index) { - this.tuple = index.getTupleExpr(); - indexQNode = ((Projection) tuple).getArg(); - sSet = getQNodes(indexQNode); - - } - - public void meet(Filter node) { - - Set<QueryModelNode> eSet = getQNodes(node); - Set<QueryModelNode> compSet = Sets.difference(eSet, sSet); - - // if index contains filter node and it hasn't already been moved, - // move it down - // query tree just above position of statement pattern nodes found - // in both query tree - // and index (assuming that SPBubbleDownVisitor has already been - // called) - if (sSet.contains(node.getCondition()) && !bubbledFilters.contains(node.getCondition())) { - FilterBubbleDownVisitor fbdv = new FilterBubbleDownVisitor((Filter) node.clone(), compSet); - node.visit(fbdv); - bubbledFilters.add(node.getCondition()); - // checks if filter correctly placed, and if it has been, - // removes old copy of filter - if (fbdv.filterPlaced()) { - - QueryModelNode pNode = node.getParentNode(); - TupleExpr cNode = node.getArg(); - pNode.replaceChildNode(node, cNode); - - - super.meetNode(pNode); - } - super.meet(node); - - } else { - super.meet(node); - } - } - } - - // iterates through the query tree and attempts to match subtrees with - // index. When a match is - // found, the subtree is replaced by an ExternalTupleSet formed from the - // index. Pre-condition for - // calling this method is that both SPBubbleDownVisitor and - // FilterBubbleManager have been called - // to position the StatementPatterns and Filters. - private static class SubsetEqualsVisitor extends QueryModelVisitorBase<RuntimeException> { - - private TupleExpr tuple; - private QueryModelNode indexQNode; - private ExternalTupleSet set; - private Set<QueryModelNode> sSet = Sets.newHashSet(); - - public SubsetEqualsVisitor(ExternalTupleSet index) { - this.tuple = index.getTupleExpr(); - this.set = index; - indexQNode = ((Projection) tuple).getArg(); - sSet = getQNodes(indexQNode); - - } - - public void meet(Join node) { - - Set<QueryModelNode> eSet = getQNodes(node); - - if (eSet.containsAll(sSet) && !(node.getRightArg() instanceof BindingSetAssignment)) { - -// System.out.println("Eset is " + eSet + " and sSet is " + sSet); - - if (eSet.equals(sSet)) { - node.replaceWith(set); - return; - } else { - if (node.getLeftArg() instanceof StatementPattern && sSet.size() == 1) { - if(sSet.contains(node.getLeftArg())) { - node.setLeftArg(set); - } else if(sSet.contains(node.getRightArg())) { - node.setRightArg(set); - } else { - return; - } - } - else { - super.meet(node); - } - } - } else if (eSet.containsAll(sSet)) { - - super.meet(node); - - } - - } - //TODO might need to include BindingSetAssignment Condition here - //to account for index consisting of only filter and BindingSetAssignment nodes - public void meet(Filter node) { - - Set<QueryModelNode> eSet = getQNodes(node); - - if (eSet.containsAll(sSet)) { - - if (eSet.equals(sSet)) { - node.replaceWith(set); - return; - } else { - super.meet(node); - } - } - } - } - - // visitor which determines whether a query is valid (i.e. it does not - // contain nodes other than - // Projection, Join, Filter, StatementPattern ) - private static class ValidQueryVisitor extends QueryModelVisitorBase<RuntimeException> { - - private boolean isValid = true; - - public boolean isValid() { - return isValid; - } - - public void meet(Projection node) { - node.getArg().visit(this); - } - - public void meet(Filter node) { - node.getArg().visit(this); - } - - - - - - public void meetNode(QueryModelNode node) { - - if (!((node instanceof Join) || (node instanceof StatementPattern) || (node instanceof BindingSetAssignment) || (node instanceof Var))) { - isValid = false; - return; - - } else{ - super.meetNode(node); - } - } - - } - - // repositions ExternalTuples above StatementPatterns within query tree - private static class ExtTupleExchangeVisitor extends QueryModelVisitorBase<RuntimeException> { - - private Set<QueryModelNode> extTuples; - - public ExtTupleExchangeVisitor(Set<QueryModelNode> extTuples) { - this.extTuples = extTuples; - } - - public void meet(Join queryNode) { - - // if query tree contains external tuples and they are not - // positioned above statement pattern node - // reposition - if (this.extTuples.size() > 0 && !(queryNode.getRightArg() instanceof ExternalTupleSet) - && !(queryNode.getRightArg() instanceof BindingSetAssignment)) { - QNodeExchanger qnev = new QNodeExchanger((QueryModelNode) queryNode.getRightArg(), this.extTuples); - queryNode.visit(qnev); - queryNode.setRightArg((TupleExpr)qnev.getReplaced()); - super.meet(queryNode); - } else { - super.meet(queryNode); - } - - } - - } - - private static class ExternalTupleCollector extends QueryModelVisitorBase<RuntimeException> { - - private Set<QueryModelNode> eSet = new HashSet<QueryModelNode>(); - - @Override - public void meetNode(QueryModelNode node) throws RuntimeException { - if (node instanceof ExternalTupleSet) { - eSet.add(node); - } - super.meetNode(node); - } - - public Set<QueryModelNode> getExtTup() { - return eSet; - } - - } - - private static class FilterCollector extends QueryModelVisitorBase<RuntimeException> { - - private List<QueryModelNode> filterList = Lists.newArrayList(); - - public List<QueryModelNode> getFilters() { - return filterList; - } - - @Override - public void meet(Filter node) { - filterList.add(node.getCondition()); - super.meet(node); - } - - } - - private static void organizeBSAs(QueryModelNode node) { - - BindingSetAssignmentCollector bsac = new BindingSetAssignmentCollector(); - node.visit(bsac); - - if (bsac.containsBSAs()) { - Set<QueryModelNode> bsaSet = bsac.getBindingSetAssignments(); - BindingSetAssignmentExchangeVisitor bsaev = new BindingSetAssignmentExchangeVisitor(bsaSet); - node.visit(bsaev); - } - } - - // repositions ExternalTuples above StatementPatterns within query tree - private static class BindingSetAssignmentExchangeVisitor extends QueryModelVisitorBase<RuntimeException> { - - private Set<QueryModelNode> bsas; - - public BindingSetAssignmentExchangeVisitor(Set<QueryModelNode> bsas) { - this.bsas = bsas; - } - - public void meet(Join queryNode) { - - // if query tree contains external tuples and they are not - // positioned above statement pattern node - // reposition - if (this.bsas.size() > 0 && !(queryNode.getRightArg() instanceof BindingSetAssignment)) { - QNodeExchanger qnev = new QNodeExchanger((QueryModelNode) queryNode.getRightArg(), bsas); - queryNode.visit(qnev); - queryNode.replaceChildNode(queryNode.getRightArg(), qnev.getReplaced()); - super.meet(queryNode); - } else { - super.meet(queryNode); - } - - } - - } - - - public static class BindingSetAssignmentCollector extends QueryModelVisitorBase<RuntimeException> { - - private Set<QueryModelNode> bindingSetList = Sets.newHashSet(); - - public Set<QueryModelNode> getBindingSetAssignments() { - return bindingSetList; - } - - public boolean containsBSAs() { - return (bindingSetList.size() > 0); - } - - @Override - public void meet(BindingSetAssignment node) { - bindingSetList.add(node); - super.meet(node); - } - - } - - // TODO insert BindingSetAssignments at bottom of query tree --this approach assumes - // BindingSetAssignments always removed during creation of ExternalTupleSets within - // query. There may be cases where this precondition does not hold (all BindingSetAssignments - // not removed). For now assuming it always holds. - -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c12f58f4/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalSail.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalSail.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalSail.java deleted file mode 100644 index 772ffa4..0000000 --- a/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalSail.java +++ /dev/null @@ -1,86 +0,0 @@ -package mvm.rya.indexing.external; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import info.aduna.iteration.CloseableIteration; - -import org.openrdf.model.ValueFactory; -import org.openrdf.query.BindingSet; -import org.openrdf.query.Dataset; -import org.openrdf.query.QueryEvaluationException; -import org.openrdf.query.algebra.Projection; -import org.openrdf.query.algebra.QueryRoot; -import org.openrdf.query.algebra.TupleExpr; -import org.openrdf.sail.Sail; -import org.openrdf.sail.SailConnection; -import org.openrdf.sail.SailException; -import org.openrdf.sail.helpers.SailBase; -import org.openrdf.sail.helpers.SailConnectionWrapper; - -public class ExternalSail extends SailBase { - private final Sail s; - private final ExternalProcessor processor; - - public ExternalSail(Sail s, ExternalProcessor processor) { - this.s = s; - this.processor = processor; - } - - @Override - protected SailConnection getConnectionInternal() throws SailException { - return new ProcessingSailConnection(); - } - - @Override - public boolean isWritable() throws SailException { - return s.isWritable(); - } - - @Override - public ValueFactory getValueFactory() { - return s.getValueFactory(); - } - - @Override - protected void shutDownInternal() throws SailException { - s.shutDown(); - } - - private class ProcessingSailConnection extends SailConnectionWrapper { - - public ProcessingSailConnection() throws SailException { - super(s.getConnection()); - } - - @Override - public CloseableIteration<? extends BindingSet, QueryEvaluationException> evaluate(TupleExpr tupleExpr, Dataset dataset, - BindingSet bindings, boolean includeInferred) throws SailException { - if ((tupleExpr instanceof Projection) || (tupleExpr instanceof QueryRoot)) { - TupleExpr processedExpression = processor.process(tupleExpr); - return super.evaluate(processedExpression, dataset, bindings, includeInferred); - } else { - return super.evaluate(tupleExpr, dataset, bindings, includeInferred); - } - - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c12f58f4/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalSailExample.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalSailExample.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalSailExample.java deleted file mode 100644 index 082dd99..0000000 --- a/extras/indexing/src/main/java/mvm/rya/indexing/external/ExternalSailExample.java +++ /dev/null @@ -1,124 +0,0 @@ -package mvm.rya.indexing.external; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import java.util.List; - -import mvm.rya.indexing.external.tupleSet.AccumuloIndexSet; -import mvm.rya.indexing.external.tupleSet.ExternalTupleSet; - -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.mock.MockInstance; -import org.openrdf.model.URI; -import org.openrdf.model.impl.LiteralImpl; -import org.openrdf.model.impl.URIImpl; -import org.openrdf.model.vocabulary.RDF; -import org.openrdf.model.vocabulary.RDFS; -import org.openrdf.query.QueryLanguage; -import org.openrdf.query.algebra.helpers.QueryModelTreePrinter; -import org.openrdf.query.parser.ParsedQuery; -import org.openrdf.query.parser.sparql.SPARQLParser; -import org.openrdf.query.resultio.sparqlxml.SPARQLResultsXMLWriter; -import org.openrdf.repository.sail.SailRepository; -import org.openrdf.repository.sail.SailRepositoryConnection; -import org.openrdf.sail.Sail; -import org.openrdf.sail.memory.MemoryStore; - -import com.google.common.collect.Lists; - -public class ExternalSailExample { - - public static void main(String[] args) throws Exception { - - Sail s = new MemoryStore(); - SailRepository repo = new SailRepository(s); - repo.initialize(); - SailRepositoryConnection conn = repo.getConnection(); - - URI sub = new URIImpl("uri:entity"); - URI subclass = new URIImpl("uri:class"); - URI obj = new URIImpl("uri:obj"); - URI talksTo = new URIImpl("uri:talksTo"); - - conn.add(sub, RDF.TYPE, subclass); - conn.add(sub, RDFS.LABEL, new LiteralImpl("label")); - conn.add(sub, talksTo, obj); - - URI sub2 = new URIImpl("uri:entity2"); - URI subclass2 = new URIImpl("uri:class2"); - URI obj2 = new URIImpl("uri:obj2"); - - conn.add(sub2, RDF.TYPE, subclass2); - conn.add(sub2, RDFS.LABEL, new LiteralImpl("label2")); - conn.add(sub2, talksTo, obj2); - - // TODO Auto-generated method stub - String indexSparqlString = ""// - + "SELECT ?e ?l ?c " // - + "{" // - + " ?e a ?c . "// - + " ?e <http://www.w3.org/2000/01/rdf-schema#label> ?l "// - + "}";// - - conn.prepareTupleQuery(QueryLanguage.SPARQL, indexSparqlString).evaluate(new SPARQLResultsXMLWriter(System.out)); - - SPARQLParser sp = new SPARQLParser(); - ParsedQuery pq = sp.parseQuery(indexSparqlString, null); - System.out.println(pq); - - List<ExternalTupleSet> index = Lists.newArrayList(); - - Connector accCon = new MockInstance().getConnector("root", "".getBytes()); - String tablename = "table"; - accCon.tableOperations().create(tablename); - index.add(new AccumuloIndexSet(indexSparqlString, conn, accCon, tablename)); - - String queryString = ""// - + "SELECT ?e ?c ?l ?o " // - + "{" // - + " ?e a ?c . "// - + " ?e <http://www.w3.org/2000/01/rdf-schema#label> ?l . "// - + " ?e <uri:talksTo> ?o . "// - + "}";// - - conn.prepareTupleQuery(QueryLanguage.SPARQL, queryString).evaluate(new SPARQLResultsXMLWriter(System.out)); - - pq = sp.parseQuery(queryString, null); - QueryModelTreePrinter mp = new QueryModelTreePrinter(); - pq.getTupleExpr().visit(mp); - System.out.println(mp.getTreeString()); - System.out.println(pq.getTupleExpr()); - - System.out.println("++++++++++++"); - ExternalProcessor processor = new ExternalProcessor(index); - System.out.println(processor.process(pq.getTupleExpr())); - - System.out.println("----------------"); - Sail processingSail = new ExternalSail(s, processor); - SailRepository smartSailRepo = new SailRepository(processingSail); - smartSailRepo.initialize(); - - smartSailRepo.getConnection().prepareTupleQuery(QueryLanguage.SPARQL, queryString).evaluate(new SPARQLResultsXMLWriter(System.out)); - - } - -}
