Repository: marmotta Updated Branches: refs/heads/develop 3753fe060 -> 8429f690d
first working implementation of MARMOTTA-536 Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/8429f690 Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/8429f690 Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/8429f690 Branch: refs/heads/develop Commit: 8429f690dfd5835c6dd8e0e3e5a048df5edaa44d Parents: 3753fe0 Author: Sebastian Schaffert <[email protected]> Authored: Tue Sep 16 13:10:47 2014 +0200 Committer: Sebastian Schaffert <[email protected]> Committed: Tue Sep 16 13:10:47 2014 +0200 ---------------------------------------------------------------------- .../kiwi/sparql/builder/ExtensionFinder.java | 57 ++++++++++++++++++++ .../kiwi/sparql/builder/SQLBuilder.java | 46 ++++++++++++++-- .../evaluation/KiWiEvaluationStrategyImpl.java | 43 +++++++++++++++ .../kiwi/sparql/test/KiWiSparqlJoinTest.java | 19 +++++++ .../marmotta/kiwi/sparql/test/query25.sparql | 22 ++++++++ .../marmotta/kiwi/sparql/test/query26.sparql | 26 +++++++++ .../marmotta/kiwi/sparql/test/query27.sparql | 25 +++++++++ 7 files changed, 235 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/8429f690/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/builder/ExtensionFinder.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/builder/ExtensionFinder.java b/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/builder/ExtensionFinder.java new file mode 100644 index 0000000..e1c8f4d --- /dev/null +++ b/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/builder/ExtensionFinder.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.marmotta.kiwi.sparql.builder; + +import org.openrdf.query.algebra.Extension; +import org.openrdf.query.algebra.ExtensionElem; +import org.openrdf.query.algebra.TupleExpr; +import org.openrdf.query.algebra.Var; +import org.openrdf.query.algebra.helpers.QueryModelVisitorBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** +* Find the offset and limit values in a tuple expression +* +* @author Sebastian Schaffert ([email protected]) +*/ +public class ExtensionFinder extends QueryModelVisitorBase<RuntimeException> { + + private static Logger log = LoggerFactory.getLogger(ExtensionFinder.class); + + List<ExtensionElem> elements = new ArrayList<>(); + + public ExtensionFinder(TupleExpr expr) { + expr.visit(this); + } + + @Override + public void meet(Extension node) throws RuntimeException { + for(ExtensionElem elem : node.getElements()) { + if(elem.getExpr() instanceof Var && ((Var) elem.getExpr()).getName().equals(elem.getName())) { + log.debug("ignoring self-aliasing of variable {}", elem.getName()); + } else { + elements.add(elem); + } + } + super.meet(node); + } +} http://git-wip-us.apache.org/repos/asf/marmotta/blob/8429f690/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/builder/SQLBuilder.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/builder/SQLBuilder.java b/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/builder/SQLBuilder.java index 14a7c06..8ed4781 100644 --- a/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/builder/SQLBuilder.java +++ b/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/builder/SQLBuilder.java @@ -19,6 +19,7 @@ package org.apache.marmotta.kiwi.sparql.builder; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; +import org.apache.marmotta.commons.collections.CollectionUtils; import org.apache.marmotta.commons.util.DateUtils; import org.apache.marmotta.kiwi.model.rdf.KiWiNode; import org.apache.marmotta.kiwi.persistence.KiWiDialect; @@ -124,6 +125,7 @@ public class SQLBuilder { private long limit = -1; private List<OrderElem> orderby; + private List<ExtensionElem> extensions; /** * A map for mapping the SPARQL variable names to internal names used for constructing SQL aliases. @@ -237,7 +239,7 @@ public class SQLBuilder { } private void prepareBuilder() throws UnsatisfiableQueryException { - Preconditions.checkArgument(query instanceof Order || query instanceof Group || query instanceof LeftJoin ||query instanceof Join || query instanceof Filter || query instanceof StatementPattern || query instanceof Distinct || query instanceof Slice || query instanceof Reduced); + Preconditions.checkArgument(query instanceof Extension || query instanceof Order || query instanceof Group || query instanceof LeftJoin ||query instanceof Join || query instanceof Filter || query instanceof StatementPattern || query instanceof Distinct || query instanceof Slice || query instanceof Reduced); // collect all patterns in a list, using depth-first search over the join @@ -255,6 +257,9 @@ public class SQLBuilder { // find the ordering orderby = new OrderFinder(query).elements; + // find extensions (BIND) + extensions = new ExtensionFinder(query).elements; + // find all variables occurring in the patterns and create a map to map them to // field names in the database query; each variable will have one or several field names, // one for each pattern it occurs in; field names are constructed automatically by a counter @@ -289,6 +294,21 @@ public class SQLBuilder { } } + // add all extensions to the variable list so they are properly considered in projections and clauses + for(ExtensionElem ext : extensions) { + Var v = new Var(ext.getName()); + if (variableNames.get(v) == null) { + variableNames.put(v, "V" + (++variableCount)); + queryVariables.put(v, new LinkedList<String>()); + queryVariableIds.put(v, new LinkedList<String>()); + } + if (hasNodeCondition(v, query)) { + queryVariables.get(v).add(evaluateExpression(ext.getExpr(), OPTypes.ANY)); + } + queryVariableIds.get(v).add(evaluateExpression(ext.getExpr(), OPTypes.ANY)); + + } + // find context restrictions of patterns and match them with potential restrictions given in the // dataset (MARMOTTA-340) for(SQLFragment f : fragments) { @@ -628,7 +648,13 @@ public class SQLBuilder { if(orderby.size() > 0) { orderClause.append("ORDER BY "); for(Iterator<OrderElem> it = orderby.iterator(); it.hasNext(); ) { - orderClause.append(evaluateExpression(it.next().getExpr(), OPTypes.VALUE)); + OrderElem elem = it.next(); + orderClause.append(evaluateExpression(elem.getExpr(), OPTypes.VALUE)); + if(elem.isAscending()) { + orderClause.append(" ASC"); + } else { + orderClause.append(" DESC"); + } if(it.hasNext()) { orderClause.append(", "); } @@ -657,7 +683,7 @@ public class SQLBuilder { - private String evaluateExpression(ValueExpr expr, OPTypes optype) { + private String evaluateExpression(ValueExpr expr, final OPTypes optype) { if(expr instanceof And) { return "(" + evaluateExpression(((And) expr).getLeftArg(), optype) + " AND " + evaluateExpression(((And) expr).getRightArg(), optype) + ")"; } else if(expr instanceof Or) { @@ -790,6 +816,13 @@ public class SQLBuilder { default: throw new IllegalArgumentException("unsupported value type: " + optype); } } + } else if(expr instanceof Coalesce) { + return "COALESCE(" + CollectionUtils.fold(((Coalesce) expr).getArguments(), new CollectionUtils.StringSerializer<ValueExpr>() { + @Override + public String serialize(ValueExpr valueExpr) { + return evaluateExpression(valueExpr, optype); + } + },", ") + ")"; } else if(expr instanceof FunctionCall) { FunctionCall fc = (FunctionCall)expr; @@ -862,6 +895,13 @@ public class SQLBuilder { private boolean hasNodeCondition(Var v, TupleExpr expr) { if(expr instanceof Filter) { return hasNodeCondition(v, ((UnaryTupleOperator) expr).getArg()) || hasNodeCondition(v, ((Filter) expr).getCondition()); + } else if(expr instanceof Extension) { + for(ExtensionElem elem : ((Extension) expr).getElements()) { + if(hasNodeCondition(v, elem.getExpr())) { + return true; + } + } + return hasNodeCondition(v,((Extension) expr).getArg()); } else if(expr instanceof Order) { for(OrderElem elem : ((Order) expr).getElements()) { if(hasNodeCondition(v, elem.getExpr())) { http://git-wip-us.apache.org/repos/asf/marmotta/blob/8429f690/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/evaluation/KiWiEvaluationStrategyImpl.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/evaluation/KiWiEvaluationStrategyImpl.java b/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/evaluation/KiWiEvaluationStrategyImpl.java index 0538d4d..8cd1a15 100644 --- a/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/evaluation/KiWiEvaluationStrategyImpl.java +++ b/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/evaluation/KiWiEvaluationStrategyImpl.java @@ -68,6 +68,35 @@ public class KiWiEvaluationStrategyImpl extends EvaluationStrategyImpl{ } @Override + public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(Extension order, BindingSet bindings) throws QueryEvaluationException { + if(Thread.currentThread().isInterrupted()) { + throw new QueryEvaluationException("SPARQL evaluation has already been cancelled"); + } + + if(isSupported(order)) { + log.debug("applying KiWi EXTENSION optimizations on SPARQL query ..."); + + try { + return new ExceptionConvertingIteration<BindingSet, QueryEvaluationException>(connection.evaluateNative(order, bindings, dataset)) { + @Override + protected QueryEvaluationException convert(Exception e) { + return new QueryEvaluationException(e); + } + }; + } catch (SQLException e) { + throw new QueryEvaluationException(e.getMessage(),e); + } catch (IllegalArgumentException e) { + throw new QueryEvaluationException(e.getMessage(),e); + } catch (InterruptedException e) { + throw new QueryInterruptedException(e.getMessage()); + } + } else { + return super.evaluate(order, bindings); + } + } + + + @Override public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(Order order, BindingSet bindings) throws QueryEvaluationException { if(Thread.currentThread().isInterrupted()) { throw new QueryEvaluationException("SPARQL evaluation has already been cancelled"); @@ -262,6 +291,13 @@ public class KiWiEvaluationStrategyImpl extends EvaluationStrategyImpl{ return isSupported(((LeftJoin) expr).getLeftArg()) && isSupported(((LeftJoin) expr).getRightArg()) && isSupported(((LeftJoin)expr).getCondition()); } else if(expr instanceof Filter) { return isSupported(((Filter) expr).getArg()) && isSupported(((Filter) expr).getCondition()); + } else if(expr instanceof Extension) { + for(ExtensionElem elem : ((Extension) expr).getElements()) { + if(!isSupported(elem.getExpr())) { + return false; + } + } + return isSupported(((Extension) expr).getArg()); } else if(expr instanceof StatementPattern) { return true; } else if(expr instanceof Slice) { @@ -299,6 +335,13 @@ public class KiWiEvaluationStrategyImpl extends EvaluationStrategyImpl{ private boolean isSupported(ValueExpr expr) { if(expr == null) { return true; + } else if(expr instanceof Coalesce) { + for(ValueExpr e : ((Coalesce) expr).getArguments()) { + if(!isSupported(e)) { + return false; + } + } + return true; } else if(expr instanceof Compare) { return isSupported(((Compare) expr).getLeftArg()) && isSupported(((Compare) expr).getRightArg()); } else if(expr instanceof MathExpr) { http://git-wip-us.apache.org/repos/asf/marmotta/blob/8429f690/libraries/kiwi/kiwi-sparql/src/test/java/org/apache/marmotta/kiwi/sparql/test/KiWiSparqlJoinTest.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-sparql/src/test/java/org/apache/marmotta/kiwi/sparql/test/KiWiSparqlJoinTest.java b/libraries/kiwi/kiwi-sparql/src/test/java/org/apache/marmotta/kiwi/sparql/test/KiWiSparqlJoinTest.java index 8be3bac..17fa358 100644 --- a/libraries/kiwi/kiwi-sparql/src/test/java/org/apache/marmotta/kiwi/sparql/test/KiWiSparqlJoinTest.java +++ b/libraries/kiwi/kiwi-sparql/src/test/java/org/apache/marmotta/kiwi/sparql/test/KiWiSparqlJoinTest.java @@ -271,6 +271,25 @@ public class KiWiSparqlJoinTest { } + // simple group by + @Test + public void testQuery25() throws Exception { + testQuery("query25.sparql"); + } + + // bind/coalesce + @Test + public void testQuery27() throws Exception { + testQuery("query27.sparql"); + } + + + // simple union + @Test + public void testQuery26() throws Exception { + testQuery("query26.sparql"); + } + // INSERT/UPDATE @Test public void testUpdate01() throws Exception { http://git-wip-us.apache.org/repos/asf/marmotta/blob/8429f690/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query25.sparql ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query25.sparql b/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query25.sparql new file mode 100644 index 0000000..36f92f8 --- /dev/null +++ b/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query25.sparql @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +PREFIX foaf: <http://xmlns.com/foaf/0.1/> + +SELECT ?p (count(?friend) AS ?friends) WHERE { + ?p foaf:knows ?friend +} GROUP BY ?p \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/8429f690/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query26.sparql ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query26.sparql b/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query26.sparql new file mode 100644 index 0000000..9bf0a06 --- /dev/null +++ b/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query26.sparql @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +PREFIX foaf: <http://xmlns.com/foaf/0.1/> + +SELECT ?p ?name ?likes WHERE { + ?p foaf:name ?name . + { ?p foaf:knows ?likes } + UNION + { ?p foaf:interest ?likes } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/8429f690/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query27.sparql ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query27.sparql b/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query27.sparql new file mode 100644 index 0000000..405a29e --- /dev/null +++ b/libraries/kiwi/kiwi-sparql/src/test/resources/org/apache/marmotta/kiwi/sparql/test/query27.sparql @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +PREFIX foaf: <http://xmlns.com/foaf/0.1/> + +SELECT ?p ?name ?likes WHERE { + ?p foaf:name ?name . + OPTIONAL { ?p foaf:knows ?knows } + OPTIONAL { ?p foaf:interest ?interest } + BIND (COALESCE(?knows,?interest) AS ?likes) +} \ No newline at end of file
