http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/accumulo/mr/fileinput/RyaBatchWriterInputTool.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/accumulo/mr/fileinput/RyaBatchWriterInputTool.java b/extras/indexing/src/main/java/mvm/rya/accumulo/mr/fileinput/RyaBatchWriterInputTool.java new file mode 100644 index 0000000..d85ffd8 --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/accumulo/mr/fileinput/RyaBatchWriterInputTool.java @@ -0,0 +1,242 @@ +package mvm.rya.accumulo.mr.fileinput; + +/* + * #%L + * mvm.rya.indexing.accumulo + * %% + * Copyright (C) 2014 Rya + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.io.IOException; +import java.io.StringReader; + +import mvm.rya.accumulo.mr.RyaOutputFormat; +import mvm.rya.accumulo.mr.StatementWritable; +import mvm.rya.accumulo.mr.utils.MRUtils; +import mvm.rya.indexing.accumulo.ConfigUtils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.log4j.Logger; +import org.openrdf.model.Resource; +import org.openrdf.model.Statement; +import org.openrdf.model.URI; +import org.openrdf.model.Value; +import org.openrdf.model.ValueFactory; +import org.openrdf.model.impl.ValueFactoryImpl; +import org.openrdf.rio.ParserConfig; +import org.openrdf.rio.RDFFormat; +import org.openrdf.rio.RDFHandlerException; +import org.openrdf.rio.RDFParseException; +import org.openrdf.rio.RDFParser; +import org.openrdf.rio.Rio; +import org.openrdf.rio.helpers.RDFHandlerBase; + +import com.google.common.base.Preconditions; + +/** + * Take large ntrips files and use MapReduce to ingest into other indexing + */ +public class RyaBatchWriterInputTool extends Configured implements Tool { + private static final Logger logger = Logger.getLogger(RyaBatchWriterInputTool.class); + + @Override + public int run(final String[] args) throws Exception { + String userName = null; + String pwd = null; + String instance = null; + String zk = null; + String format = null; + + final Configuration conf = getConf(); + // conf + zk = conf.get(MRUtils.AC_ZK_PROP, zk); + instance = conf.get(MRUtils.AC_INSTANCE_PROP, instance); + userName = conf.get(MRUtils.AC_USERNAME_PROP, userName); + pwd = conf.get(MRUtils.AC_PWD_PROP, pwd); + format = conf.get(MRUtils.FORMAT_PROP, RDFFormat.NTRIPLES.getName()); + + String auths = conf.get(MRUtils.AC_CV_PROP, ""); + + conf.set(MRUtils.FORMAT_PROP, format); + Preconditions.checkNotNull(zk, MRUtils.AC_ZK_PROP + " not set"); + Preconditions.checkNotNull(instance, MRUtils.AC_INSTANCE_PROP + " not set"); + Preconditions.checkNotNull(userName, MRUtils.AC_USERNAME_PROP + " not set"); + Preconditions.checkNotNull(pwd, MRUtils.AC_PWD_PROP + " not set"); + + // map the config values to free text configure values + conf.set(ConfigUtils.CLOUDBASE_ZOOKEEPERS, zk); + conf.set(ConfigUtils.CLOUDBASE_INSTANCE, instance); + conf.set(ConfigUtils.CLOUDBASE_USER, userName); + conf.set(ConfigUtils.CLOUDBASE_PASSWORD, pwd); + conf.set(ConfigUtils.CLOUDBASE_AUTHS, auths); + + final String inputDir = args[0]; + + String tablePrefix = conf.get(MRUtils.TABLE_PREFIX_PROPERTY, null); + Preconditions.checkNotNull(tablePrefix, MRUtils.TABLE_PREFIX_PROPERTY + " not set"); + + String docTextTable = tablePrefix + "text"; + conf.set(ConfigUtils.FREE_TEXT_DOC_TABLENAME, docTextTable); + + String docTermTable = tablePrefix + "terms"; + conf.set(ConfigUtils.FREE_TEXT_TERM_TABLENAME, docTermTable); + + String geoTable = tablePrefix + "geo"; + conf.set(ConfigUtils.GEO_TABLENAME, geoTable); + + logger.info("Loading data into tables[rya, freetext, geo]"); + logger.info("Loading data into tables[" + docTermTable + " " + docTextTable + " " + geoTable + "]"); + + Job job = new Job(new Configuration(conf), "Batch Writer load data into Rya Core and Indexing Tables"); + job.setJarByClass(this.getClass()); + + // setting long job + Configuration jobConf = job.getConfiguration(); + jobConf.setBoolean("mapred.map.tasks.speculative.execution", false); + + jobConf.setInt("mapred.task.timeout", 1000 * 60 * 60 * 24); // timeout after 1 day + + job.setInputFormatClass(TextInputFormat.class); + + job.setMapperClass(ParseNtripsMapper.class); + + job.setNumReduceTasks(0); + + // Use Rya Output Format + job.setOutputFormatClass(RyaOutputFormat.class); + job.setOutputKeyClass(NullWritable.class); + job.setOutputValueClass(StatementWritable.class); + job.setMapOutputKeyClass(NullWritable.class); + job.setMapOutputValueClass(StatementWritable.class); + + TextInputFormat.setInputPaths(job, new Path(inputDir)); + + job.waitForCompletion(true); + + return 0; + } + + public static void main(String[] args) throws Exception { + ToolRunner.run(new Configuration(), new RyaBatchWriterInputTool(), args); + } + + public static class ParseNtripsMapper extends Mapper<LongWritable, Text, Writable, Statement> { + private static final Logger logger = Logger.getLogger(ParseNtripsMapper.class); + + private RDFParser parser; + private RDFFormat rdfFormat; + + @Override + protected void setup(final Context context) throws IOException, InterruptedException { + super.setup(context); + Configuration conf = context.getConfiguration(); + + final ValueFactory vf = new ValueFactoryImpl(); + + String rdfFormatName = conf.get(MRUtils.FORMAT_PROP); + checkNotNull(rdfFormatName, "Rdf format cannot be null"); + rdfFormat = RDFFormat.valueOf(rdfFormatName); + + String namedGraphString = conf.get(MRUtils.NAMED_GRAPH_PROP); + checkNotNull(namedGraphString, MRUtils.NAMED_GRAPH_PROP + " cannot be null"); + + final Resource namedGraph = vf.createURI(namedGraphString); + + parser = Rio.createParser(rdfFormat); + parser.setParserConfig(new ParserConfig(true, true, true, RDFParser.DatatypeHandling.VERIFY)); + parser.setRDFHandler(new RDFHandlerBase() { + @Override + public void handleStatement(Statement statement) throws RDFHandlerException { + Statement output; + if (rdfFormat.equals(RDFFormat.NTRIPLES)) { + output = new ConextStatementWrapper(statement, namedGraph); + } else { + output = statement; + } + try { + context.write(NullWritable.get(), new StatementWritable(output)); + } catch (IOException e) { + logger.error("Error writing statement", e); + throw new RDFHandlerException(e); + } catch (InterruptedException e) { + logger.error("Error writing statement", e); + throw new RDFHandlerException(e); + } + } + + }); + } + + @Override + public void map(LongWritable key, Text value, Context output) throws IOException, InterruptedException { + String rdf = value.toString(); + try { + parser.parse(new StringReader(rdf), ""); + } catch (RDFParseException e) { + logger.error("Line[" + rdf + "] cannot be formatted with format[" + rdfFormat + "]. Exception[" + e.getMessage() + + "]", e); + } catch (Exception e) { + logger.error("error during map", e); + throw new IOException("Exception occurred parsing triple[" + rdf + "]", e); + } + } + } + + @SuppressWarnings("serial") + private static class ConextStatementWrapper implements Statement { + private Statement statementWithoutConext; + private Resource context; + + public ConextStatementWrapper(Statement statementWithoutConext, Resource context) { + this.statementWithoutConext = statementWithoutConext; + this.context = context; + } + + @Override + public Resource getSubject() { + return statementWithoutConext.getSubject(); + } + + @Override + public URI getPredicate() { + return statementWithoutConext.getPredicate(); + } + + @Override + public Value getObject() { + return statementWithoutConext.getObject(); + } + + @Override + public Resource getContext() { + return context; + } + } + +}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/accumulo/precompQuery/AccumuloPrecompQueryIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/accumulo/precompQuery/AccumuloPrecompQueryIndexer.java b/extras/indexing/src/main/java/mvm/rya/accumulo/precompQuery/AccumuloPrecompQueryIndexer.java new file mode 100644 index 0000000..9612bce --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/accumulo/precompQuery/AccumuloPrecompQueryIndexer.java @@ -0,0 +1,306 @@ +package mvm.rya.accumulo.precompQuery; + +import info.aduna.iteration.CloseableIteration; + +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.hadoop.io.Text; +import org.openrdf.query.Binding; +import org.openrdf.query.BindingSet; +import org.openrdf.query.QueryEvaluationException; +import org.openrdf.query.algebra.evaluation.QueryBindingSet; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import mvm.rya.indexing.PrecompQueryIndexer; +import mvm.rya.indexing.external.tupleSet.AccumuloIndexSet.AccValueFactory; + +public class AccumuloPrecompQueryIndexer implements PrecompQueryIndexer { + + + private Connector accCon; + private String tableName; + private Map<String, AccValueFactory> bindings; + + + + public AccumuloPrecompQueryIndexer(Connector accCon, String tableName) { + this.accCon = accCon; + this.tableName = tableName; + } + + + @Override + public void storeBindingSet(BindingSet bs) throws IOException { + // TODO Auto-generated method stub + + } + + @Override + public void storeBindingSets(Collection<BindingSet> bindingSets) throws IOException, IllegalArgumentException { + // TODO Auto-generated method stub + + } + + @Override + public CloseableIteration<BindingSet, QueryEvaluationException> queryPrecompJoin(List<String> varOrder, + String localityGroup, Map<String, AccValueFactory> bindings, Map<String, org.openrdf.model.Value> valMap, Collection<BindingSet> bsConstraints) + throws QueryEvaluationException, TableNotFoundException { + + + final int prefixLength = Integer.parseInt(varOrder.remove(varOrder.size()-1)); + final Iterator<Entry<Key,Value>> accIter; + final HashMultimap<Range,BindingSet> map = HashMultimap.create(); + final List<BindingSet> extProdList = Lists.newArrayList(); + final Map<String, AccValueFactory> bindingMap = bindings; + final List<String> order = varOrder; + final BatchScanner bs = accCon.createBatchScanner(tableName, new Authorizations(), 10); + final Set<Range> ranges = Sets.newHashSet(); + + + + bs.fetchColumnFamily(new Text(localityGroup)); + + //process bindingSet and constant constraints + for (BindingSet bSet : bsConstraints) { + StringBuffer rangePrefix = new StringBuffer(); + int i = 0; + + for (String b : order) { + + if (i >= prefixLength) { + break; + } + + if (b.startsWith("-const-")) { + String val = bindings.get(b).create(valMap.get(b)); + rangePrefix.append(val); + rangePrefix.append("\u0000"); + } else { + + Binding v = bSet.getBinding(b); + if (v == null) { + throw new IllegalStateException("Binding set can't have null value!"); + } + String val = bindings.get(b).create(bSet.getValue(b)); + rangePrefix.append(val); + rangePrefix.append("\u0000"); + + } + + i++; + + } + if (rangePrefix.length() > 0) { + String prefixWithOutNull = rangePrefix.deleteCharAt(rangePrefix.length() - 1).toString(); + String prefixWithNull = prefixWithOutNull + "\u0001"; + Range r = new Range(new Key(prefixWithOutNull), true, new Key(prefixWithNull), false); + map.put(r, bSet); + ranges.add(r); + } else if (bSet.size() > 0) { + extProdList.add(bSet); + } + } + + //constant constraints and no bindingSet constraints + //add range of entire table if no constant constraints and + //bsConstraints consists of single, empty set (occurs when AIS is + //first node evaluated in query) + if (ranges.isEmpty() && bsConstraints.size() > 0) { + + if (prefixLength > 0) { + StringBuffer rangePrefix = new StringBuffer(); + + int i = 0; + for (String b : order) { + if (i >= prefixLength) { + break; + } + if (b.startsWith("-const-")) { + String val = bindings.get(b).create(valMap.get(b)); + rangePrefix.append(val); + rangePrefix.append("\u0000"); + } + i++; + } + + String prefixWithOutNull = rangePrefix.deleteCharAt(rangePrefix.length() - 1).toString(); + String prefixWithNull = prefixWithOutNull + "\u0001"; + Range r = new Range(new Key(prefixWithOutNull), true, new Key(prefixWithNull), false); + ranges.add(r); + + } else { // no constant or bindingSet constraints + ranges.add(new Range("", true, "~", false)); + } + } + + if (ranges.size() == 0) { + accIter = null; + } else { + bs.setRanges(ranges); + accIter = bs.iterator(); + } + + + return new CloseableIteration<BindingSet, QueryEvaluationException>() { + + @Override + public void remove() throws QueryEvaluationException { + throw new UnsupportedOperationException(); + } + + private Iterator<BindingSet> inputSet = null; + private QueryBindingSet currentSolutionBs = null; + private boolean hasNextCalled = false; + private boolean isEmpty = false; + + + + @Override + public BindingSet next() throws QueryEvaluationException { + QueryBindingSet bs = new QueryBindingSet(); + + if (hasNextCalled) { + hasNextCalled = false; + if (inputSet != null) { + bs.addAll(inputSet.next()); + } + bs.addAll(currentSolutionBs); + } else if (isEmpty) { + throw new NoSuchElementException(); + } else { + if (this.hasNext()) { + hasNextCalled = false; + if (inputSet != null) { + bs.addAll(inputSet.next()); + } + bs.addAll(currentSolutionBs); + } else { + throw new NoSuchElementException(); + } + } + + return bs; + } + + @Override + public boolean hasNext() throws QueryEvaluationException { + + if(accIter == null ) { + isEmpty = true; + return false; + } + + if (!hasNextCalled && !isEmpty) { + while (accIter.hasNext() || (inputSet != null && inputSet.hasNext())) { + + if(inputSet != null && inputSet.hasNext()) { + hasNextCalled = true; + return true; + } + + + Key k = accIter.next().getKey(); + final String[] s = k.getRow().toString().split("\u0000"); + + StringBuilder rangePrefix = new StringBuilder(); + // TODO Assuming that order specifies order of variables + // commmon to + // bindingSet passed in and variables in index table + // --size is equal to + + for (int i = 0; i < prefixLength; i++) { + rangePrefix.append(s[i]); + rangePrefix.append("\u0000"); + } + + // TODO I need to remember what the type was! + currentSolutionBs = new QueryBindingSet(); + int i = 0; + for (String b : order) { + if (b.startsWith("-const")) { + i++; + } else { + final String v = s[i]; + currentSolutionBs.addBinding(b, bindingMap.get(b).create(v)); + i++; + } + + } + //check to see if bindingSet constraints exist + if (map.size() > 0) { + String prefixWithOutNull = rangePrefix.deleteCharAt(rangePrefix.length() - 1).toString(); + String prefixWithNull = prefixWithOutNull + "\u0001"; + Range r = new Range(new Key(prefixWithOutNull), true, new Key(prefixWithNull), false); + inputSet = map.get(r).iterator(); + if (!inputSet.hasNext()) { + continue; + } else { + hasNextCalled = true; + return true; + } // check to see if binding set constraints exist, but no common vars + } else if (extProdList.size() > 0) { + inputSet = extProdList.iterator(); + hasNextCalled = true; + return true; + }else { //no bindingsSet constraints--only constant constraints or none + hasNextCalled = true; + return true; + } + } + + isEmpty = true; + return false; + + } else if (isEmpty) { + return false; + } else { + return true; + } + + } + + @Override + public void close() throws QueryEvaluationException { + bs.close(); + } + + }; + } + + + + @Override + public void flush() throws IOException { + // TODO Auto-generated method stub + + } + + @Override + public void close() throws IOException { + // TODO Auto-generated method stub + + } + + + + + + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/indexing/DocIdIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/DocIdIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/DocIdIndexer.java new file mode 100644 index 0000000..354c025 --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/DocIdIndexer.java @@ -0,0 +1,33 @@ +package mvm.rya.indexing; + +import info.aduna.iteration.CloseableIteration; + +import java.io.Closeable; +import java.io.Flushable; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import mvm.rya.indexing.accumulo.entity.StarQuery; +import mvm.rya.indexing.external.tupleSet.AccumuloIndexSet.AccValueFactory; + +import org.apache.accumulo.core.client.TableNotFoundException; +import org.openrdf.model.Value; +import org.openrdf.query.BindingSet; +import org.openrdf.query.QueryEvaluationException; +import org.openrdf.query.algebra.TupleExpr; + +public interface DocIdIndexer extends Closeable { + + + + public abstract CloseableIteration<BindingSet, QueryEvaluationException> queryDocIndex(StarQuery query, + Collection<BindingSet> constraints) throws TableNotFoundException, QueryEvaluationException; + + + + @Override + public abstract void close() throws IOException; + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java b/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java new file mode 100644 index 0000000..8d258b4 --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java @@ -0,0 +1,338 @@ +package mvm.rya.indexing; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import mvm.rya.accumulo.AccumuloRdfConfiguration; +import mvm.rya.indexing.IndexingFunctionRegistry.FUNCTION_TYPE; +import mvm.rya.indexing.accumulo.ConfigUtils; +import mvm.rya.indexing.accumulo.freetext.AccumuloFreeTextIndexer; +import mvm.rya.indexing.accumulo.freetext.FreeTextTupleSet; +import mvm.rya.indexing.accumulo.geo.GeoMesaGeoIndexer; +import mvm.rya.indexing.accumulo.geo.GeoTupleSet; +import mvm.rya.indexing.accumulo.temporal.AccumuloTemporalIndexer; +import mvm.rya.indexing.accumulo.temporal.TemporalTupleSet; +import mvm.rya.indexing.mongodb.MongoGeoIndexer; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.commons.lang.Validate; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.geotools.feature.SchemaException; +import org.openrdf.model.Resource; +import org.openrdf.model.URI; +import org.openrdf.model.Value; +import org.openrdf.model.ValueFactory; +import org.openrdf.model.impl.URIImpl; +import org.openrdf.model.impl.ValueFactoryImpl; +import org.openrdf.query.BindingSet; +import org.openrdf.query.Dataset; +import org.openrdf.query.algebra.And; +import org.openrdf.query.algebra.Filter; +import org.openrdf.query.algebra.FunctionCall; +import org.openrdf.query.algebra.Join; +import org.openrdf.query.algebra.LeftJoin; +import org.openrdf.query.algebra.QueryModelNode; +import org.openrdf.query.algebra.StatementPattern; +import org.openrdf.query.algebra.TupleExpr; +import org.openrdf.query.algebra.ValueConstant; +import org.openrdf.query.algebra.ValueExpr; +import org.openrdf.query.algebra.Var; +import org.openrdf.query.algebra.evaluation.QueryOptimizer; +import org.openrdf.query.algebra.helpers.QueryModelVisitorBase; + +import com.google.common.collect.Lists; + +public class FilterFunctionOptimizer implements QueryOptimizer, Configurable { + + private ValueFactory valueFactory = new ValueFactoryImpl(); + + private Configuration conf; + private GeoIndexer geoIndexer; + private FreeTextIndexer freeTextIndexer; + private TemporalIndexer temporalIndexer; + private boolean init = false; + + + public FilterFunctionOptimizer() { + } + + + public FilterFunctionOptimizer(AccumuloRdfConfiguration conf) throws AccumuloException, AccumuloSecurityException, + TableNotFoundException, IOException, SchemaException, TableExistsException { + this.conf = conf; + init(); + } + + //setConf initializes FilterFunctionOptimizer so reflection can be used + //to create optimizer in RdfCloudTripleStoreConnection + @Override + public void setConf(Configuration conf) { + this.conf = conf; + init(); + } + + + private void init() { + if (!init) { + if (ConfigUtils.getUseMongo(conf)) { + this.geoIndexer = new MongoGeoIndexer(); + geoIndexer.setConf(conf); + } else { + this.geoIndexer = new GeoMesaGeoIndexer(); + geoIndexer.setConf(conf); + this.freeTextIndexer = new AccumuloFreeTextIndexer(); + freeTextIndexer.setConf(conf); + this.temporalIndexer = new AccumuloTemporalIndexer(); + temporalIndexer.setConf(conf); + init = true; + } + } + } + + @Override + public void optimize(TupleExpr tupleExpr, Dataset dataset, BindingSet bindings) { + // find variables used in property and resource based searches: + SearchVarVisitor searchVars = new SearchVarVisitor(); + tupleExpr.visit(searchVars); + // rewrites for property searches: + processPropertySearches(tupleExpr, searchVars.searchProperties); + + } + + + + private void processPropertySearches(TupleExpr tupleExpr, Collection<Var> searchProperties) { + MatchStatementVisitor matchStatements = new MatchStatementVisitor(searchProperties); + tupleExpr.visit(matchStatements); + for (StatementPattern matchStatement: matchStatements.matchStatements) { + Var subject = matchStatement.getSubjectVar(); + if (subject.hasValue() && !(subject.getValue() instanceof Resource)) + throw new IllegalArgumentException("Query error: Found " + subject.getValue() + ", expected an URI or BNode"); + Validate.isTrue(subject.hasValue() || subject.getName() != null); + Validate.isTrue(!matchStatement.getObjectVar().hasValue() && matchStatement.getObjectVar().getName() != null); + buildQuery(tupleExpr, matchStatement); + } + + } + + + private void buildQuery(TupleExpr tupleExpr, StatementPattern matchStatement) { + //If our IndexerExpr (to be) is the rhs-child of LeftJoin, we can safely make that a Join: + // the IndexerExpr will (currently) not return results that can deliver unbound variables. + //This optimization should probably be generalized into a LeftJoin -> Join optimizer under certain conditions. Until that + // has been done, this code path at least takes care of queries generated by OpenSahara SparqTool that filter on OPTIONAL + // projections. E.g. summary~'full text search' (summary is optional). See #379 + if (matchStatement.getParentNode() instanceof LeftJoin) { + LeftJoin leftJoin = (LeftJoin)matchStatement.getParentNode(); + if (leftJoin.getRightArg() == matchStatement && leftJoin.getCondition() == null) + matchStatement.getParentNode().replaceWith(new Join(leftJoin.getLeftArg(), leftJoin.getRightArg())); + } + FilterFunction fVisitor = new FilterFunction(matchStatement.getObjectVar().getName()); + tupleExpr.visit(fVisitor); + List<IndexingExpr> results = Lists.newArrayList(); + for(int i = 0; i < fVisitor.func.size(); i++){ + results.add(new IndexingExpr(fVisitor.func.get(i), matchStatement, fVisitor.args.get(i))); + } + removeMatchedPattern(tupleExpr, matchStatement, new IndexerExprReplacer(results)); + + } + + //find vars contained in filters + private static class SearchVarVisitor extends QueryModelVisitorBase<RuntimeException> { + + private final Collection<Var> searchProperties = new ArrayList<Var>(); + + @Override + public void meet(FunctionCall fn) { + URI fun = new URIImpl(fn.getURI()); + Var result = IndexingFunctionRegistry.getResultVarFromFunctionCall(fun, fn.getArgs()); + if (result != null && !searchProperties.contains(result)) + searchProperties.add(result); + } + } + + //find StatementPatterns containing filter variables + private static class MatchStatementVisitor extends QueryModelVisitorBase<RuntimeException> { + private final Collection<Var> propertyVars; + private final Collection<Var> usedVars = new ArrayList<Var>(); + private final List<StatementPattern> matchStatements = new ArrayList<StatementPattern>(); + + public MatchStatementVisitor(Collection<Var> propertyVars) { + this.propertyVars = propertyVars; + } + + @Override public void meet(StatementPattern statement) { + Var object = statement.getObjectVar(); + if (propertyVars.contains(object)) + if (usedVars.contains(object)) + throw new IllegalArgumentException("Illegal search, variable is used multiple times as object: " + object.getName()); + else { + usedVars.add(object); + matchStatements.add(statement); + } + } + } + + private abstract class AbstractEnhanceVisitor extends QueryModelVisitorBase<RuntimeException> { + final String matchVar; + List<URI> func = Lists.newArrayList(); + List<Value[]> args = Lists.newArrayList(); + + public AbstractEnhanceVisitor(String matchVar) { + this.matchVar = matchVar; + } + + protected void addFilter(URI uri, Value[] values) { + func.add(uri); + args.add(values); + } + } + + //create indexing expression for each filter matching var in filter StatementPattern + //replace old filter condition with true condition + private class FilterFunction extends AbstractEnhanceVisitor { + public FilterFunction(String matchVar) { + super(matchVar); + } + + @Override + public void meet(FunctionCall call) { + URI fnUri = valueFactory.createURI(call.getURI()); + Var resultVar = IndexingFunctionRegistry.getResultVarFromFunctionCall(fnUri, call.getArgs()); + if (resultVar != null && resultVar.getName().equals(matchVar)) { + addFilter(valueFactory.createURI(call.getURI()), extractArguments(matchVar, call)); + if (call.getParentNode() instanceof Filter || call.getParentNode() instanceof And || call.getParentNode() instanceof LeftJoin) + call.replaceWith(new ValueConstant(valueFactory.createLiteral(true))); + else + throw new IllegalArgumentException("Query error: Found " + call + " as part of an expression that is too complex"); + } + } + + + private Value[] extractArguments(String matchName, FunctionCall call) { + Value args[] = new Value[call.getArgs().size() - 1]; + int argI = 0; + for (int i = 0; i != call.getArgs().size(); ++i) { + ValueExpr arg = call.getArgs().get(i); + if (argI == i && arg instanceof Var && matchName.equals(((Var)arg).getName())) + continue; + if (arg instanceof ValueConstant) + args[argI] = ((ValueConstant)arg).getValue(); + else if (arg instanceof Var && ((Var)arg).hasValue()) + args[argI] = ((Var)arg).getValue(); + else + throw new IllegalArgumentException("Query error: Found " + arg + ", expected a Literal, BNode or URI"); + ++argI; + } + return args; + } + + @Override + public void meet(Filter filter) { + //First visit children, then condition (reverse of default): + filter.getArg().visit(this); + filter.getCondition().visit(this); + } + } + + private void removeMatchedPattern(TupleExpr tupleExpr, StatementPattern pattern, TupleExprReplacer replacer) { + List<TupleExpr> indexTuples = replacer.createReplacement(pattern); + if (indexTuples.size() > 1) { + VarExchangeVisitor vev = new VarExchangeVisitor(pattern); + tupleExpr.visit(vev); + Join join = new Join(indexTuples.remove(0), indexTuples.remove(0)); + for (TupleExpr geo : indexTuples) { + join = new Join(join, geo); + } + pattern.replaceWith(join); + } else if (indexTuples.size() == 1) { + pattern.replaceWith(indexTuples.get(0)); + pattern.setParentNode(null); + } else { + throw new IllegalStateException("Must have at least one replacement for matched StatementPattern."); + } + } + + private interface TupleExprReplacer { + List<TupleExpr> createReplacement(TupleExpr org); + } + + + //replace each filter pertinent StatementPattern with corresponding index expr + private class IndexerExprReplacer implements TupleExprReplacer { + private final List<IndexingExpr> indxExpr; + private FUNCTION_TYPE type; + + public IndexerExprReplacer(List<IndexingExpr> indxExpr) { + this.indxExpr = indxExpr; + URI func = indxExpr.get(0).getFunction(); + this.type = IndexingFunctionRegistry.getFunctionType(func); + } + + @Override + public List<TupleExpr> createReplacement(TupleExpr org) { + List<TupleExpr> indexTuples = Lists.newArrayList(); + switch (type) { + case GEO: + for (IndexingExpr indx : indxExpr) { + indexTuples.add(new GeoTupleSet(indx, geoIndexer)); + } + break; + case FREETEXT: + for (IndexingExpr indx : indxExpr) { + indexTuples.add(new FreeTextTupleSet(indx, freeTextIndexer)); + } + break; + case TEMPORAL: + for (IndexingExpr indx : indxExpr) { + indexTuples.add(new TemporalTupleSet(indx, temporalIndexer)); + } + break; + default: + throw new IllegalArgumentException("Incorrect type!"); + + } + return indexTuples; + } + } + + + private static class VarExchangeVisitor extends QueryModelVisitorBase<RuntimeException> { + + private final StatementPattern exchangeVar; + + public VarExchangeVisitor(StatementPattern sp) { + this.exchangeVar = sp; + } + + @Override + public void meet(Join node) { + QueryModelNode lNode = node.getLeftArg(); + if (lNode instanceof StatementPattern) { + exchangeVar.replaceWith(lNode); + node.setLeftArg(exchangeVar); + } else { + super.meet(node); + } + } + } + + + + + + + @Override + public Configuration getConf() { + return conf; + } + + + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/indexing/FreeTextIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/FreeTextIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/FreeTextIndexer.java new file mode 100644 index 0000000..646fe5b --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/FreeTextIndexer.java @@ -0,0 +1,61 @@ +package mvm.rya.indexing; + +/* + * #%L + * mvm.rya.indexing.accumulo + * %% + * Copyright (C) 2014 Rya + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import info.aduna.iteration.CloseableIteration; + +import java.io.IOException; +import java.util.Set; + +import mvm.rya.api.persist.index.RyaSecondaryIndexer; + +import org.openrdf.model.Statement; +import org.openrdf.model.URI; +import org.openrdf.query.QueryEvaluationException; + +/** + * A repository to store, index, and retrieve {@link Statement}s based on freetext features. + */ +public interface FreeTextIndexer extends RyaSecondaryIndexer { + + /** + * Query the Free Text Index with specific constraints. A <code>null</code> or empty parameters imply no constraint. + * + * @param query + * the query to perform + * @param contraints + * the constraints on the statements returned + * @return the set of statements that meet the query and other constraints. + * @throws IOException + */ + public abstract CloseableIteration<Statement, QueryEvaluationException> queryText(String query, StatementContraints contraints) throws IOException; + + /** + * @return the set of predicates indexed by the indexer. + */ + public abstract Set<URI> getIndexablePredicates(); + + @Override + public abstract void flush() throws IOException; + + @Override + public abstract void close() throws IOException; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/indexing/GeoIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/GeoIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/GeoIndexer.java new file mode 100644 index 0000000..9ef2cce --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/GeoIndexer.java @@ -0,0 +1,200 @@ +package mvm.rya.indexing; + +/* + * #%L + * mvm.rya.indexing.accumulo + * %% + * Copyright (C) 2014 Rya + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import info.aduna.iteration.CloseableIteration; + +import java.io.IOException; +import java.util.Set; + +import mvm.rya.api.persist.index.RyaSecondaryIndexer; + +import org.openrdf.model.Statement; +import org.openrdf.model.URI; +import org.openrdf.query.QueryEvaluationException; + +import com.vividsolutions.jts.geom.Geometry; + +/** + * A repository to store, index, and retrieve {@link Statement}s based on geospatial features. + */ +public interface GeoIndexer extends RyaSecondaryIndexer { + /** + * Returns statements that contain a geometry that is equal to the queried {@link Geometry} and meet the {@link StatementContraints}. + * + * <p> + * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM): + * <ul> + * <li> + * "Two geometries are topologically equal if their interiors intersect and no part of the interior or boundary of one geometry intersects the exterior of the other" + * <li>"A is equal to B if A is within B and A contains B" + * </ul> + * + * @param query + * the queried geometry + * @param contraints + * the {@link StatementContraints} + * @return + */ + public abstract CloseableIteration<Statement, QueryEvaluationException> queryEquals(Geometry query, StatementContraints contraints); + + /** + * Returns statements that contain a geometry that is disjoint to the queried {@link Geometry} and meet the {@link StatementContraints}. + * + * <p> + * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM): + * <ul> + * <li>"A and B are disjoint if they have no point in common. They form a set of disconnected geometries." + * <li>"A and B are disjoint if A does not intersect B" + * </ul> + * + * @param query + * the queried geometry + * @param contraints + * the {@link StatementContraints} + * @return + */ + public abstract CloseableIteration<Statement, QueryEvaluationException> queryDisjoint(Geometry query, StatementContraints contraints); + + /** + * Returns statements that contain a geometry that Intersects the queried {@link Geometry} and meet the {@link StatementContraints}. + * + * <p> + * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM): + * <ul> + * <li>"a intersects b: geometries a and b have at least one point in common." + * <li>"not Disjoint" + * </ul> + * + * + * @param query + * the queried geometry + * @param contraints + * the {@link StatementContraints} + * @return + */ + public abstract CloseableIteration<Statement, QueryEvaluationException> queryIntersects(Geometry query, StatementContraints contraints); + + /** + * Returns statements that contain a geometry that Touches the queried {@link Geometry} and meet the {@link StatementContraints}. + * + * <p> + * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM): + * <ul> + * <li>"a touches b, they have at least one boundary point in common, but no interior points." + * </ul> + * + * + * @param query + * the queried geometry + * @param contraints + * the {@link StatementContraints} + * @return + */ + public abstract CloseableIteration<Statement, QueryEvaluationException> queryTouches(Geometry query, StatementContraints contraints); + + /** + * Returns statements that contain a geometry that crosses the queried {@link Geometry} and meet the {@link StatementContraints}. + * + * <p> + * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM): + * <ul> + * <li> + * "a crosses b, they have some but not all interior points in common (and the dimension of the intersection is less than that of at least one of them)." + * </ul> + * + * @param query + * the queried geometry + * @param contraints + * the {@link StatementContraints} + * @return + */ + public abstract CloseableIteration<Statement, QueryEvaluationException> queryCrosses(Geometry query, StatementContraints contraints); + + /** + * Returns statements that contain a geometry that is Within the queried {@link Geometry} and meet the {@link StatementContraints}. + * + * <p> + * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM): + * <ul> + * <li>"a is within b, a lies in the interior of b" + * <li>Same as: "Contains(b,a)" + * </ul> + * + * + * @param query + * the queried geometry + * @param contraints + * the {@link StatementContraints} + * @return + */ + public abstract CloseableIteration<Statement, QueryEvaluationException> queryWithin(Geometry query, StatementContraints contraints); + + /** + * Returns statements that contain a geometry that Contains the queried {@link Geometry} and meet the {@link StatementContraints}. + * + * <p> + * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM): + * <ul> + * <li>b is within a. Geometry b lies in the interior of a. Another definition: + * "a 'contains' b iff no points of b lie in the exterior of a, and at least one point of the interior of b lies in the interior of a" + * <li>Same: Within(b,a) + * </ul> + * + * + * @param query + * the queried geometry + * @param contraints + * the {@link StatementContraints} + * @return + */ + public abstract CloseableIteration<Statement, QueryEvaluationException> queryContains(Geometry query, StatementContraints contraints); + + /** + * Returns statements that contain a geometry that Overlaps the queried {@link Geometry} and meet the {@link StatementContraints}. + * + * <p> + * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM): + * <ul> + * <li>a crosses b, they have some but not all interior points in common (and the dimension of the intersection is less than that of at + * least one of them). + * </ul> + * + * + * @param query + * the queried geometry + * @param contraints + * the {@link StatementContraints} + * @return + */ + public abstract CloseableIteration<Statement, QueryEvaluationException> queryOverlaps(Geometry query, StatementContraints contraints); + + /** + * @return the set of predicates indexed by the indexer. + */ + public abstract Set<URI> getIndexablePredicates(); + + @Override + public abstract void flush() throws IOException; + + @Override + public abstract void close() throws IOException; +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/ExternalIndexMatcher.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/ExternalIndexMatcher.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/ExternalIndexMatcher.java new file mode 100644 index 0000000..eb0a397 --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/ExternalIndexMatcher.java @@ -0,0 +1,18 @@ +package mvm.rya.indexing.IndexPlanValidator; + +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import mvm.rya.indexing.external.tupleSet.ExternalTupleSet; + +import org.openrdf.query.algebra.TupleExpr; + +public interface ExternalIndexMatcher { + + + public Iterator<TupleExpr> getIndexedTuples(); + + + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/GeneralizedExternalProcessor.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/GeneralizedExternalProcessor.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/GeneralizedExternalProcessor.java new file mode 100644 index 0000000..114fff3 --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/GeneralizedExternalProcessor.java @@ -0,0 +1,731 @@ +package mvm.rya.indexing.IndexPlanValidator; + + + +/* + * #%L + * mvm.rya.rya.indexing + * %% + * Copyright (C) 2014 - 2015 Rya + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import mvm.rya.indexing.external.QueryVariableNormalizer.VarCollector; +import mvm.rya.indexing.external.tupleSet.ExternalTupleSet; + +import org.openrdf.query.algebra.BindingSetAssignment; +import org.openrdf.query.algebra.Filter; +import org.openrdf.query.algebra.Join; +import org.openrdf.query.algebra.Projection; +import org.openrdf.query.algebra.QueryModelNode; +import org.openrdf.query.algebra.StatementPattern; +import org.openrdf.query.algebra.TupleExpr; +import org.openrdf.query.algebra.Var; +import org.openrdf.query.algebra.helpers.QueryModelVisitorBase; +import org.openrdf.query.algebra.helpers.StatementPatternCollector; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +/** + * Processes a {@link TupleExpr} and replaces sets of elements in the tree with {@link ExternalTupleSet} objects. + */ +public class GeneralizedExternalProcessor { + + + /** + * Iterates through list of normalized indexes and replaces all subtrees of query which match index with index. + * + * @param query + * @return TupleExpr + */ + public static TupleExpr process(TupleExpr query, List<ExternalTupleSet> indexSet) { + + boolean indexPlaced = false; + TupleExpr rtn = query.clone(); + + + //TODO optimization: turn on when testing done + QueryNodeCount qnc = new QueryNodeCount(); + rtn.visit(qnc); + + if(qnc.getNodeCount()/2 < indexSet.size()) { + return null; + } + + + //move BindingSetAssignment Nodes out of the way + organizeBSAs(rtn); + + + // test to see if query contains no other nodes + // than filter, join, projection, and statement pattern and + // test whether query contains duplicate StatementPatterns and filters + if (isTupleValid(rtn)) { + + for (ExternalTupleSet index : indexSet) { + + // test to see if index contains at least one StatementPattern, + // that StatementPatterns are unique, + // and that all variables found in filters occur in some + // StatementPattern + if (isTupleValid(index.getTupleExpr())) { + + ExternalTupleSet eTup = (ExternalTupleSet) index.clone(); + SPBubbleDownVisitor indexVistor = new SPBubbleDownVisitor(eTup); + rtn.visit(indexVistor); + FilterBubbleManager fbmv = new FilterBubbleManager(eTup); + rtn.visit(fbmv); + SubsetEqualsVisitor subIndexVis = new SubsetEqualsVisitor(eTup, rtn); + rtn.visit(subIndexVis); + indexPlaced = subIndexVis.indexPlaced(); + if(!indexPlaced) { + break; + } + + } + + } + if(indexPlaced) { +// if(indexSet.size() == 3) { +// System.out.println("IndexSet is " + indexSet); +// System.out.println("Tuple is " + rtn); +// } + return rtn; + } else { +// if(indexSet.size() == 3) { +// System.out.println("IndexSet is " + indexSet); +// } +// + return null; + } + + } else { + throw new IllegalArgumentException("Invalid Query."); + } + } + + + + + + // determines whether query is valid, which requires that a + // query must contain a StatementPattern, not contain duplicate + // Statement Patterns or Filters, not be comprised of only Projection, + // Join, StatementPattern, and Filter nodes, and that any variable + // appearing in a Filter must appear in a StatementPattern. + private static boolean isTupleValid(QueryModelNode node) { + + ValidQueryVisitor vqv = new ValidQueryVisitor(); + node.visit(vqv); + + Set<String> spVars = getVarNames(getQNodes("sp", node)); + + if (vqv.isValid() && (spVars.size() > 0)) { + + FilterCollector fvis = new FilterCollector(); + node.visit(fvis); + List<QueryModelNode> fList = fvis.getFilters(); + return (fList.size() == Sets.newHashSet(fList).size() && getVarNames(fList).size() <= spVars.size()); + + } else { + return false; + } + } + + private static Set<QueryModelNode> getQNodes(QueryModelNode queryNode) { + Set<QueryModelNode> rtns = new HashSet<QueryModelNode>(); + + StatementPatternCollector spc = new StatementPatternCollector(); + queryNode.visit(spc); + rtns.addAll(spc.getStatementPatterns()); + + FilterCollector fvis = new FilterCollector(); + queryNode.visit(fvis); + rtns.addAll(fvis.getFilters()); + + ExternalTupleCollector eVis = new ExternalTupleCollector(); + queryNode.visit(eVis); + rtns.addAll(eVis.getExtTup()); + + return rtns; + } + + private static Set<QueryModelNode> getQNodes(String node, QueryModelNode queryNode) { + + if (node.equals("sp")) { + Set<QueryModelNode> eSet = new HashSet<QueryModelNode>(); + StatementPatternCollector spc = new StatementPatternCollector(); + queryNode.visit(spc); + List<StatementPattern> spList = spc.getStatementPatterns(); + eSet.addAll(spList); + // returns empty set if list contains duplicate StatementPatterns + if (spList.size() > eSet.size()) { + return Sets.newHashSet(); + } else { + return eSet; + } + } else if (node.equals("filter")) { + + FilterCollector fvis = new FilterCollector(); + queryNode.visit(fvis); + + return Sets.newHashSet(fvis.getFilters()); + } else { + + throw new IllegalArgumentException("Invalid node type."); + } + } + + // moves StatementPatterns in query that also occur in index to bottom of + // query tree. + private static class SPBubbleDownVisitor extends QueryModelVisitorBase<RuntimeException> { + + private TupleExpr tuple; + private QueryModelNode indexQNode; + private Set<QueryModelNode> sSet = Sets.newHashSet(); + + public SPBubbleDownVisitor(ExternalTupleSet index) { + + this.tuple = index.getTupleExpr(); + indexQNode = ((Projection) tuple).getArg(); + sSet = getQNodes("sp", indexQNode); + + } + + public void meet(Projection node) { + // moves external tuples above statement patterns before attempting + // to bubble down index statement patterns found in query tree + + organizeExtTuples(node); + + super.meet(node); + } + + public void meet(Join node) { + // if right node contained in index, move it to bottom of query tree + if (sSet.contains(node.getRightArg())) { + + Set<QueryModelNode> eSet = getQNodes("sp", node); + Set<QueryModelNode> compSet = Sets.difference(eSet, sSet); + + if (eSet.containsAll(sSet)) { + + QNodeExchanger qne = new QNodeExchanger(node.getRightArg(), compSet); + node.visit(qne); + node.replaceChildNode(node.getRightArg(), qne.getReplaced()); + + super.meet(node); + } + return; + } + // if left node contained in index, move it to bottom of query tree + else if (sSet.contains(node.getLeftArg())) { + + Set<QueryModelNode> eSet = getQNodes("sp", node); + Set<QueryModelNode> compSet = Sets.difference(eSet, sSet); + + if (eSet.containsAll(sSet)) { + + QNodeExchanger qne = new QNodeExchanger(node.getLeftArg(), compSet); + node.visit(qne); + node.replaceChildNode(node.getLeftArg(), qne.getReplaced()); + + super.meet(node); + } + return; + + } else { + super.meet(node); + } + + } + + // moves all ExternalTupleSets in query tree above remaining + // StatementPatterns + private static void organizeExtTuples(QueryModelNode node) { + + ExternalTupleCollector eVis = new ExternalTupleCollector(); + node.visit(eVis); + + ExtTupleExchangeVisitor oev = new ExtTupleExchangeVisitor(eVis.getExtTup()); + node.visit(oev); + } + + } + + // given a replacement QueryModelNode and compSet, this visitor replaces the + // first + // element in the query tree that occurs in compSet with replacement and + // returns + // the element that was replaced. + private static class QNodeExchanger extends QueryModelVisitorBase<RuntimeException> { + + private QueryModelNode toBeReplaced; + private QueryModelNode replacement; + private Set<QueryModelNode> compSet; + + public QNodeExchanger(QueryModelNode replacement, Set<QueryModelNode> compSet) { + this.replacement = replacement; + this.toBeReplaced = replacement; + this.compSet = compSet; + } + + public QueryModelNode getReplaced() { + return toBeReplaced; + } + + public void meet(Join node) { + + if (compSet.contains(node.getRightArg())) { + this.toBeReplaced = node.getRightArg(); + node.replaceChildNode(node.getRightArg(), replacement); + return; + } else if (compSet.contains(node.getLeftArg())) { + this.toBeReplaced = node.getLeftArg(); + node.replaceChildNode(node.getLeftArg(), replacement); + return; + } else { + super.meet(node); + } + + } + + } + + // moves filter that occurs in both query and index down the query tree so + // that that it is positioned + // above statement patterns associated with index. Precondition for calling + // this method is that + // SPBubbleDownVisitor has been called to position index StatementPatterns + // within query tree. + //TODO this visitor assumes that all filters are positioned at top of query tree + //could lead to problems if filter optimizer called before external processor + private static class FilterBubbleDownVisitor extends QueryModelVisitorBase<RuntimeException> { + + private QueryModelNode filter; + private Set<QueryModelNode> compSet; + private boolean filterPlaced = false; + + public FilterBubbleDownVisitor(QueryModelNode filter, Set<QueryModelNode> compSet) { + this.filter = filter; + this.compSet = compSet; + + } + + public boolean filterPlaced() { + return filterPlaced; + } + + public void meet(Join node) { + + if (!compSet.contains(node.getRightArg())) { + // looks for placed to position filter node. if right node is + // contained in index + // and left node is statement pattern node contained in index or + // is a join, place + // filter above join. + if (node.getLeftArg() instanceof Join || !(compSet.contains(node.getLeftArg()))) { + + QueryModelNode pNode = node.getParentNode(); + ((Filter) filter).setArg(node); + pNode.replaceChildNode(node, filter); + filterPlaced = true; + + return; + } // otherwise place filter below join and above right arg + else { + ((Filter) filter).setArg(node.getRightArg()); + node.replaceChildNode(node.getRightArg(), filter); + filterPlaced = true; + return; + + } + } else if ((node.getLeftArg() instanceof StatementPattern) && !compSet.contains(node.getLeftArg())) { + + ((Filter) filter).setArg(node.getLeftArg()); + node.replaceChildNode(node.getLeftArg(), filter); + filterPlaced = true; + + return; + } else { + super.meet(node); + } + } + + } + + private static Set<String> getVarNames(Collection<QueryModelNode> nodes) { + + List<String> tempVars; + Set<String> nodeVarNames = Sets.newHashSet(); + + for (QueryModelNode s : nodes) { + tempVars = VarCollector.process(s); + for (String t : tempVars) + nodeVarNames.add(t); + } + return nodeVarNames; + + } + + // visitor which determines whether or not to reposition a filter by calling + // FilterBubbleDownVisitor + private static class FilterBubbleManager extends QueryModelVisitorBase<RuntimeException> { + + private TupleExpr tuple; + private QueryModelNode indexQNode; + private Set<QueryModelNode> sSet = Sets.newHashSet(); + private Set<QueryModelNode> bubbledFilters = Sets.newHashSet(); + + public FilterBubbleManager(ExternalTupleSet index) { + this.tuple = index.getTupleExpr(); + indexQNode = ((Projection) tuple).getArg(); + sSet = getQNodes(indexQNode); + + } + + public void meet(Filter node) { + + Set<QueryModelNode> eSet = getQNodes(node); + Set<QueryModelNode> compSet = Sets.difference(eSet, sSet); + + // if index contains filter node and it hasn't already been moved, + // move it down + // query tree just above position of statement pattern nodes found + // in both query tree + // and index (assuming that SPBubbleDownVisitor has already been + // called) + if (sSet.contains(node.getCondition()) && !bubbledFilters.contains(node.getCondition())) { + FilterBubbleDownVisitor fbdv = new FilterBubbleDownVisitor((Filter) node.clone(), compSet); + node.visit(fbdv); + bubbledFilters.add(node.getCondition()); + // checks if filter correctly placed, and if it has been, + // removes old copy of filter + if (fbdv.filterPlaced()) { + + QueryModelNode pNode = node.getParentNode(); + TupleExpr cNode = node.getArg(); + pNode.replaceChildNode(node, cNode); + + + super.meetNode(pNode); + } + super.meet(node); + + } else { + super.meet(node); + } + } + } + + // iterates through the query tree and attempts to match subtrees with + // index. When a match is + // found, the subtree is replaced by an ExternalTupleSet formed from the + // index. Pre-condition for + // calling this method is that both SPBubbleDownVisitor and + // FilterBubbleManager have been called + // to position the StatementPatterns and Filters. + private static class SubsetEqualsVisitor extends QueryModelVisitorBase<RuntimeException> { + + private TupleExpr query; + private TupleExpr tuple; + private QueryModelNode indexQNode; + private ExternalTupleSet set; + private Set<QueryModelNode> sSet = Sets.newHashSet(); + private TupleExpr temp; + private boolean indexPlaced = false; + + + public SubsetEqualsVisitor(ExternalTupleSet index, TupleExpr query) { + this.query = query; + this.tuple = index.getTupleExpr(); + this.set = index; + indexQNode = ((Projection) tuple).getArg(); + sSet = getQNodes(indexQNode); + + } + + public boolean indexPlaced() { + return indexPlaced; + } + + + public void meet(Join node) { + + Set<QueryModelNode> eSet = getQNodes(node); + + if (eSet.containsAll(sSet) && !(node.getRightArg() instanceof BindingSetAssignment)) { + +// System.out.println("Eset is " + eSet + " and sSet is " + sSet); + + if (eSet.equals(sSet)) { + node.replaceWith(set); + indexPlaced = true; + return; + } else { + if (node.getLeftArg() instanceof StatementPattern && sSet.size() == 1) { + if(sSet.contains(node.getLeftArg())) { + node.setLeftArg(set); + indexPlaced = true; + } else if(sSet.contains(node.getRightArg())) { + node.setRightArg(set); + indexPlaced = true; + } else { + return; + } + } + else { + super.meet(node); + } + } + } else if (eSet.containsAll(sSet)) { + + super.meet(node); + + } else { + return; + } + + } + //TODO might need to include BindingSetAssignment Condition here + //to account for index consisting of only filter and BindingSetAssignment nodes + public void meet(Filter node) { + + Set<QueryModelNode> eSet = getQNodes(node); + + if (eSet.containsAll(sSet)) { + + if (eSet.equals(sSet)) { + node.replaceWith(set); + indexPlaced = true; + return; + } else { + node.getArg().visit(this); + } + } + } + + + public void meet(StatementPattern node) { + return; + } + } + + // visitor which determines whether a query is valid (i.e. it does not + // contain nodes other than + // Projection, Join, Filter, StatementPattern ) + private static class ValidQueryVisitor extends QueryModelVisitorBase<RuntimeException> { + + private boolean isValid = true; + + public boolean isValid() { + return isValid; + } + + public void meet(Projection node) { + node.getArg().visit(this); + } + + public void meet(Filter node) { + node.getArg().visit(this); + } + + + + + + public void meetNode(QueryModelNode node) { + + if (!((node instanceof Join) || (node instanceof StatementPattern) || (node instanceof BindingSetAssignment) || (node instanceof Var))) { + isValid = false; + return; + + } else{ + super.meetNode(node); + } + } + + } + + // repositions ExternalTuples above StatementPatterns within query tree + private static class ExtTupleExchangeVisitor extends QueryModelVisitorBase<RuntimeException> { + + private Set<QueryModelNode> extTuples; + + public ExtTupleExchangeVisitor(Set<QueryModelNode> extTuples) { + this.extTuples = extTuples; + } + + public void meet(Join queryNode) { + + // if query tree contains external tuples and they are not + // positioned above statement pattern node + // reposition + if (this.extTuples.size() > 0 && !(queryNode.getRightArg() instanceof ExternalTupleSet) + && !(queryNode.getRightArg() instanceof BindingSetAssignment)) { + + if (queryNode.getLeftArg() instanceof ExternalTupleSet) { + QueryModelNode temp = queryNode.getLeftArg(); + queryNode.setLeftArg(queryNode.getRightArg()); + queryNode.setRightArg((TupleExpr)temp); + } else { + + QNodeExchanger qnev = new QNodeExchanger((QueryModelNode) queryNode.getRightArg(), this.extTuples); + queryNode.visit(qnev); + queryNode.replaceChildNode(queryNode.getRightArg(), qnev.getReplaced()); + super.meet(queryNode); + } + } else { + super.meet(queryNode); + } + + } + + } + + private static class ExternalTupleCollector extends QueryModelVisitorBase<RuntimeException> { + + private Set<QueryModelNode> eSet = new HashSet<QueryModelNode>(); + + @Override + public void meetNode(QueryModelNode node) throws RuntimeException { + if (node instanceof ExternalTupleSet) { + eSet.add(node); + } + super.meetNode(node); + } + + public Set<QueryModelNode> getExtTup() { + return eSet; + } + + } + + private static class FilterCollector extends QueryModelVisitorBase<RuntimeException> { + + private List<QueryModelNode> filterList = Lists.newArrayList(); + + public List<QueryModelNode> getFilters() { + return filterList; + } + + @Override + public void meet(Filter node) { + filterList.add(node.getCondition()); + super.meet(node); + } + + } + + private static void organizeBSAs(QueryModelNode node) { + + BindingSetAssignmentCollector bsac = new BindingSetAssignmentCollector(); + node.visit(bsac); + + if (bsac.containsBSAs()) { + Set<QueryModelNode> bsaSet = bsac.getBindingSetAssignments(); + BindingSetAssignmentExchangeVisitor bsaev = new BindingSetAssignmentExchangeVisitor(bsaSet); + node.visit(bsaev); + } + } + + // repositions ExternalTuples above StatementPatterns within query tree + private static class BindingSetAssignmentExchangeVisitor extends QueryModelVisitorBase<RuntimeException> { + + private Set<QueryModelNode> bsas; + + public BindingSetAssignmentExchangeVisitor(Set<QueryModelNode> bsas) { + this.bsas = bsas; + } + + public void meet(Join queryNode) { + + // if query tree contains external tuples and they are not + // positioned above statement pattern node + // reposition + if (this.bsas.size() > 0 && !(queryNode.getRightArg() instanceof BindingSetAssignment)) { + QNodeExchanger qnev = new QNodeExchanger((QueryModelNode) queryNode.getRightArg(), bsas); + queryNode.visit(qnev); + queryNode.replaceChildNode(queryNode.getRightArg(), qnev.getReplaced()); + super.meet(queryNode); + } else { + super.meet(queryNode); + } + + } + + } + + + public static class BindingSetAssignmentCollector extends QueryModelVisitorBase<RuntimeException> { + + private Set<QueryModelNode> bindingSetList = Sets.newHashSet(); + + public Set<QueryModelNode> getBindingSetAssignments() { + return bindingSetList; + } + + public boolean containsBSAs() { + return (bindingSetList.size() > 0); + } + + @Override + public void meet(BindingSetAssignment node) { + bindingSetList.add(node); + super.meet(node); + } + + } + + + + public static class QueryNodeCount extends QueryModelVisitorBase<RuntimeException> { + + private int nodeCount; + + public QueryNodeCount() { + nodeCount = 0; + } + + public int getNodeCount() { + return nodeCount; + } + + + @Override + public void meet(StatementPattern node) { + nodeCount += 1; + return; + } + + @Override + public void meet(Filter node) { + nodeCount += 1; + node.getArg().visit(this); + } + + } + + + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexListPruner.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexListPruner.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexListPruner.java new file mode 100644 index 0000000..4862a5c --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexListPruner.java @@ -0,0 +1,15 @@ +package mvm.rya.indexing.IndexPlanValidator; + +import java.util.List; + + + +import java.util.Set; + +import mvm.rya.indexing.external.tupleSet.ExternalTupleSet; + +public interface IndexListPruner { + + public Set<ExternalTupleSet> getRelevantIndices(List<ExternalTupleSet> indexList); + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexPlanValidator.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexPlanValidator.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexPlanValidator.java new file mode 100644 index 0000000..88dfce9 --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexPlanValidator.java @@ -0,0 +1,190 @@ +package mvm.rya.indexing.IndexPlanValidator; + +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +import mvm.rya.indexing.external.tupleSet.ExternalTupleSet; + +import org.openrdf.query.algebra.BindingSetAssignment; +import org.openrdf.query.algebra.Filter; +import org.openrdf.query.algebra.Join; +import org.openrdf.query.algebra.Projection; +import org.openrdf.query.algebra.StatementPattern; +import org.openrdf.query.algebra.TupleExpr; +import org.openrdf.query.algebra.helpers.QueryModelVisitorBase; + +import com.google.common.collect.Sets; + + + + +public class IndexPlanValidator implements TupleValidator { + + private boolean omitCrossProd = false; + + + public IndexPlanValidator(boolean omitCrossProd) { + this.omitCrossProd = omitCrossProd; + } + + public void setOmitCrossProd(boolean omitCrossProd) { + this.omitCrossProd = omitCrossProd; + } + + + @Override + public boolean isValid(TupleExpr te) { + + TupleValidateVisitor tv = new TupleValidateVisitor(); + te.visit(tv); + + return tv.isValid(); + } + + + + + public int getValidTupleSize(Iterator<TupleExpr> iter) { + + int size = 0; + + while(iter.hasNext()) { + if(isValid(iter.next())) { + size++; + } + } + + return size; + + } + + + + @Override + public Iterator<TupleExpr> getValidTuples(Iterator<TupleExpr> tupleIter) { + + final Iterator<TupleExpr> iter = tupleIter; + + return new Iterator<TupleExpr>() { + + private TupleExpr next = null; + private boolean hasNextCalled = false; + private boolean isEmpty = false; + + @Override + public boolean hasNext() { + + if (!hasNextCalled && !isEmpty) { + while (iter.hasNext()) { + TupleExpr temp = iter.next(); + if (isValid(temp)) { + next = temp; + hasNextCalled = true; + return true; + } + } + isEmpty = true; + return false; + } else if(isEmpty) { + return false; + }else { + return true; + } + } + + @Override + public TupleExpr next() { + + if (hasNextCalled) { + hasNextCalled = false; + return next; + } else if(isEmpty) { + throw new NoSuchElementException(); + }else { + if (this.hasNext()) { + hasNextCalled = false; + return next; + } else { + throw new NoSuchElementException(); + } + } + } + + @Override + public void remove() { + + throw new UnsupportedOperationException("Cannot delete from iterator!"); + + } + + }; + } + + private boolean isJoinValid(Join join) { + + Set<String> leftBindingNames = join.getLeftArg().getBindingNames(); + Set<String> rightBindingNames = join.getRightArg().getBindingNames(); + + + //System.out.println("Left binding names are " + leftBindingNames + " and right binding names are " + rightBindingNames); + + if (Sets.intersection(leftBindingNames, rightBindingNames).size() == 0) { + if (omitCrossProd) { + return false; + } else { + return true; + } + + } else { + if (join.getRightArg() instanceof ExternalTupleSet) { + + return ((ExternalTupleSet) join.getRightArg()).supportsBindingSet(leftBindingNames); + + } else { + return true; + } + } + + } + + public class TupleValidateVisitor extends QueryModelVisitorBase<RuntimeException> { + + private boolean isValid = true; + + public boolean isValid() { + return isValid; + } + + @Override + public void meet(Projection node) { + node.getArg().visit(this); + } + + @Override + public void meet(StatementPattern node) { + return; + } + + public void meet(BindingSetAssignment node) { + return; + } + + @Override + public void meet(Filter node) { + node.getArg().visit(this); + } + + @Override + public void meet(Join node) { + if (isJoinValid(node)) { + super.meet(node); + } else { + isValid = false; + return; + } + } + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexTupleGenerator.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexTupleGenerator.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexTupleGenerator.java new file mode 100644 index 0000000..6843025 --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexTupleGenerator.java @@ -0,0 +1,15 @@ +package mvm.rya.indexing.IndexPlanValidator; + +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import org.openrdf.query.algebra.TupleExpr; + +public interface IndexTupleGenerator { + + + public Iterator<TupleExpr> getPlans(Iterator<TupleExpr> indexPlans); + + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexedExecutionPlanGenerator.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexedExecutionPlanGenerator.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexedExecutionPlanGenerator.java new file mode 100644 index 0000000..d65b2bd --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexedExecutionPlanGenerator.java @@ -0,0 +1,187 @@ +package mvm.rya.indexing.IndexPlanValidator; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; + +import mvm.rya.indexing.external.QueryVariableNormalizer; +import mvm.rya.indexing.external.tupleSet.ExternalTupleSet; + +import org.openrdf.query.algebra.Projection; +import org.openrdf.query.algebra.TupleExpr; + +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +public class IndexedExecutionPlanGenerator implements ExternalIndexMatcher { + + private final TupleExpr query; + private List<ExternalTupleSet> normalizedIndexList; + + public IndexedExecutionPlanGenerator(TupleExpr query, List<ExternalTupleSet> indexList) { + this.query = query; + VarConstantIndexListPruner vci = new VarConstantIndexListPruner(query); + normalizedIndexList = getNormalizedIndices(vci.getRelevantIndices(indexList)); + } + + public List<ExternalTupleSet> getNormalizedIndices() { + return normalizedIndexList; + } + + + + + @Override + public Iterator<TupleExpr> getIndexedTuples() { + + ValidIndexCombinationGenerator vic = new ValidIndexCombinationGenerator(query); + final Iterator<List<ExternalTupleSet>> iter = vic.getValidIndexCombos(normalizedIndexList); + + return new Iterator<TupleExpr>() { + + private TupleExpr next = null; + private boolean hasNextCalled = false; + private boolean isEmpty = false; + + @Override + public boolean hasNext() { + + if (!hasNextCalled && !isEmpty) { + while (iter.hasNext()) { + TupleExpr temp = GeneralizedExternalProcessor.process(query, iter.next()); + if (temp != null) { + next = temp; + hasNextCalled = true; + return true; + } + } + isEmpty = true; + return false; + } else if(isEmpty) { + return false; + } else { + return true; + } + } + + @Override + public TupleExpr next() { + + if (hasNextCalled) { + hasNextCalled = false; + return next; + } else if(isEmpty) { + throw new NoSuchElementException(); + }else { + if (this.hasNext()) { + hasNextCalled = false; + return next; + } else { + throw new NoSuchElementException(); + } + + } + + } + + @Override + public void remove() { + + throw new UnsupportedOperationException("Cannot delete from iterator!"); + + } + + }; + } + + + private List<ExternalTupleSet> getNormalizedIndices(Set<ExternalTupleSet> indexSet) { + + ExternalTupleSet tempIndex; + List<ExternalTupleSet> normalizedIndexSet = Lists.newArrayList(); + + for (ExternalTupleSet e : indexSet) { + + List<TupleExpr> tupList = null; + try { + tupList = QueryVariableNormalizer.getNormalizedIndex(query, e.getTupleExpr()); + } catch (Exception e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + + for (TupleExpr te : tupList) { + + tempIndex = (ExternalTupleSet) e.clone(); + setTableMap(te, tempIndex); + setSupportedVarOrderMap(tempIndex); + tempIndex.setProjectionExpr((Projection) te); + normalizedIndexSet.add(tempIndex); + + } + + } + + return normalizedIndexSet; + } + + private void setTableMap(TupleExpr tupleMatch, ExternalTupleSet index) { + + List<String> replacementVars = Lists.newArrayList(tupleMatch.getBindingNames()); + List<String> tableVars = Lists.newArrayList(index.getTupleExpr().getBindingNames()); + + Map<String, String> tableMap = Maps.newHashMap(); + + for (int i = 0; i < tableVars.size(); i++) { + tableMap.put(replacementVars.get(i), tableVars.get(i)); + } + // System.out.println("Table map is " + tableMap); + index.setTableVarMap(tableMap); + + } + + + private void setSupportedVarOrderMap(ExternalTupleSet index) { + + Map<String, Set<String>> supportedVarOrders = Maps.newHashMap(); + BiMap<String, String> biMap = HashBiMap.create(index.getTableVarMap()).inverse(); + Map<String, Set<String>> oldSupportedVarOrders = index.getSupportedVariableOrderMap(); + + Set<String> temp = null; + Set<String> keys = oldSupportedVarOrders.keySet(); + + for (String s : keys) { + temp = oldSupportedVarOrders.get(s); + Set<String> newSet = Sets.newHashSet(); + + for (String t : temp) { + newSet.add(biMap.get(t)); + } + + String[] tempStrings = s.split("\u0000"); + String v = ""; + for(String u: tempStrings) { + if(v.length() == 0){ + v = v + biMap.get(u); + } else { + v = v + "\u0000" + biMap.get(u); + } + } + + supportedVarOrders.put(v, newSet); + + } + + index.setSupportedVariableOrderMap(supportedVarOrders); + + } + + + + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexedQueryPlanSelector.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexedQueryPlanSelector.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexedQueryPlanSelector.java new file mode 100644 index 0000000..097d37d --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexedQueryPlanSelector.java @@ -0,0 +1,13 @@ +package mvm.rya.indexing.IndexPlanValidator; + +import java.util.Iterator; +import java.util.List; + +import org.openrdf.query.algebra.TupleExpr; + +public interface IndexedQueryPlanSelector { + + public TupleExpr getThreshholdQueryPlan(Iterator<TupleExpr> tupleList, double threshhold, + double indexWeight, double commonVarWeight, double dirProdWeight); + +}
