Author: andy
Date: Sun Apr 14 17:59:16 2013
New Revision: 1467806
URL: http://svn.apache.org/r1467806
Log:
Work on optimizer (issue 432 and related issues).
Removed:
jena/Scratch/AFS/Jena-Dev/trunk/src/dev/TransformFilterEquality2.java
Modified:
jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Jena383_BadPlanOptionals.java
jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Jena432_SubQueryRewrites.java
jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Run.java
jena/Scratch/AFS/Jena-Dev/trunk/src/reports/archive/Jena154_GraphVarScope.java
Modified: jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Jena383_BadPlanOptionals.java
URL:
http://svn.apache.org/viewvc/jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Jena383_BadPlanOptionals.java?rev=1467806&r1=1467805&r2=1467806&view=diff
==============================================================================
--- jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Jena383_BadPlanOptionals.java
(original)
+++ jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Jena383_BadPlanOptionals.java Sun
Apr 14 17:59:16 2013
@@ -40,18 +40,6 @@ public class Jena383_BadPlanOptionals
* At least only in outer invokation.
*/
-// // Not covered by left ... but used at all in the right?
-// Op opRight = opleftjoin.getRight() ;
-// Set<Var> varsRight = OpVars.patternVars(opRight) ;
-// //return false ;
-//
-// // ----
-// for ( Var varEq : varsEquality)
-// if ( varsRight.contains(varEq) )
-// return false ;
-// return true ;
-// }
-
String qs = StrUtils.strjoinNL(
"PREFIX ex: <http://example.org/test#>",
"SELECT * WHERE {",
Modified: jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Jena432_SubQueryRewrites.java
URL:
http://svn.apache.org/viewvc/jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Jena432_SubQueryRewrites.java?rev=1467806&r1=1467805&r2=1467806&view=diff
==============================================================================
--- jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Jena432_SubQueryRewrites.java
(original)
+++ jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Jena432_SubQueryRewrites.java Sun
Apr 14 17:59:16 2013
@@ -18,65 +18,292 @@
package dev;
-import java.util.Set ;
-
import org.apache.jena.atlas.lib.StrUtils ;
+import org.junit.Test ;
import com.hp.hpl.jena.query.Query ;
import com.hp.hpl.jena.query.QueryFactory ;
import com.hp.hpl.jena.sparql.algebra.Algebra ;
import com.hp.hpl.jena.sparql.algebra.Op ;
-import com.hp.hpl.jena.sparql.algebra.OpVars ;
-import com.hp.hpl.jena.sparql.core.Var ;
-import com.hp.hpl.jena.sparql.sse.SSE ;
+import com.hp.hpl.jena.sparql.algebra.Transform ;
+import com.hp.hpl.jena.sparql.algebra.Transformer ;
+import com.hp.hpl.jena.sparql.algebra.op.OpLeftJoin ;
+import com.hp.hpl.jena.sparql.algebra.optimize.* ;
+import com.hp.hpl.jena.sparql.engine.main.LeftJoinClassifier ;
public class Jena432_SubQueryRewrites
{
+ // remove assignment from projection?
+
+ // JENA-432 : Filter optimization messes up when a nested select is
present in the same block
+ // JENA-383 : The query optimizer generates a suboptimal query plan in
case of nested optionals followed by a filter
+ // JENA-384 : SubstitueFilterOptimize :: Interaction of optimization
(TransformFilterEquality) and initial binding
+ // JENA-293 : Allow the filter placement optimziation to push FILTER
though BIND.
+ // JENA-294 : TransformFilterEquality does not handle starting OPTIONAL
well
+
public static void main(String... args) throws Exception
{
- if ( false )
+ // // Notes:
+ // // check TestTransformQuads.quads10,20,21,30
+ //
+ // if ( false )
+ // {
+ // Op op = SSE.parseOp("(leftjoin (bgp (?s1 ?p1 ?o1))
(project (?s2) (bgp (?s2 ?p2 ?o2))))") ;
+ // System.out.println(op) ;
+ // Set<Var> vars = OpVars.visibleVars(op) ;
+ // System.out.println(vars) ;
+ // System.exit(0) ;
+ // }
+
+ String qs1 = StrUtils.strjoinNL
+ ( "PREFIX : <http://example/> SELECT * {"
+ , " OPTIONAL { ?x :q ?o }"
+ , " FILTER(?x = :x)"
+ , " ?x :p ?o2"
+ , "}"
+ ) ;
+
+
+ String[] x = { qs1 } ;
+ for ( String qs : x )
{
- Op op = SSE.parseOp("(leftjoin (bgp (?s1 ?p1 ?o1)) (project (?s2)
(bgp (?s2 ?p2 ?o2))))") ;
+ Query query = QueryFactory.create(qs) ;
+ Op op = Algebra.compile(query) ;
+
+ if ( op instanceof OpLeftJoin )
+ LeftJoinClassifier.isLinear((OpLeftJoin)op) ;
+
+ System.out.println(query) ;
System.out.println(op) ;
- Set<Var> vars = OpVars.patternVars(op) ;
- System.out.println(vars) ;
- System.exit(0) ;
+
+ Op op1 = op ;
+ if ( false )
+ {
+ op1 = TransformScopeRename.transform(op1) ;
+ // Op op1 = Transformer.transform(new
TransformJoinStrategy(), op) ;
+ // Op op1 = Transformer.transform(new
TransformFilterDisjunction(), op) ;
+ op1 = Transformer.transform(new TransformFilterEquality(),
op1) ;
+ }
+ else
+ op1 = Algebra.optimize(op) ;
+ System.out.print(op1) ;
+ System.out.println("- - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - - - - - -") ;
+
}
- String qs1 = StrUtils.strjoinNL
- ( "SELECT *",
- "WHERE {",
- " ?test ?p1 ?o1.",
- //" FILTER ( ?test = <http://localhost/t1> || ?test =
<http://localhost/t2> )",
- //" FILTER ( ?test = <http://localhost/t1> )",
- " OPTIONAL {",
- " SELECT ?s1", // Not ?test
- " { ?s1 ?p2 ?o2 }", //" GROUP BY ?s1",
- " }" ,
- "}" ) ;
-
- // No (conditional) optimization.
- String qs2 = StrUtils.strjoinNL
- ( "SELECT *",
- "WHERE {",
- " ?test ?p1 ?o1.",
- " OPTIONAL {",
- " SELECT ?s1",
- " { ?s1 ?p2 ?o2 }", //" GROUP BY ?s1",
- " }" ,
- "}" ) ;
-
- String qs = qs2 ;
- Query query = QueryFactory.create(qs) ;
- Op op = Algebra.compile(query) ;
- System.out.println(query) ;
- System.out.println(op) ;
-// Op op1 = Transformer.transform(new TransformFilterDisjunction(), op)
;
-// Op op1 = Transformer.transform(new TransformFilterEquality(), op) ;
-
- //Op opX = SSE.parseOp("") ;
- Op op1 = Algebra.optimize(op) ;
- System.out.println(op1) ;
}
+ private Transform t_equality = new TransformFilterEquality() ;
+ private Transform t_disjunction = new TransformFilterDisjunction() ;
+ private Transform t_placement = new TransformFilterPlacement() ;
+ private Transform t_expandOneOf = new TransformExpandOneOf() ;
+
+ @Test public void test_OptEqualityJoin_01() {
+ String qs = StrUtils.strjoinNL
+ ( "PREFIX : <http://example/> SELECT * {"
+ , " OPTIONAL { ?x :q ?o }"
+ , " FILTER(?x = :x)"
+ , " ?x :p ?o2"
+ , "}"
+ ) ;
+ String ops = StrUtils.strjoinNL
+ ("(filter (= ?x <http://example/x>)"
+ , " (sequence"
+ , " (conditional"
+ , " (table unit)"
+ , " (bgp (triple ?x <http://example/q> ?o)))"
+ , " (bgp (triple ?x <http://example/p> ?o2))))"
+ ) ;
+ check(qs, ops) ;
+ }
+
+
+
+ @Test public void test_OptEqualitySubQuery_01() {
+ String qs = StrUtils.strjoinNL
+ ( "SELECT * {"
+ , " ?test ?p1 ?o1."
+ , " FILTER ( ?test = <http://localhost/t1> || ?test =
<http://localhost/t2> )"
+ , " OPTIONAL {"
+ , " SELECT ?s1"
+ , " { ?s1 ?p2 ?o2 }"
+ , " }"
+ , "}" ) ;
+ String ops = StrUtils.strjoinNL
+ ( "(disjunction"
+ , " (assign ((?test <http://localhost/t1>))"
+ , " (conditional"
+ , " (bgp (triple <http://localhost/t1> ?p1 ?o1))"
+ , " (project (?s1)"
+ , " (bgp (triple ?s1 ?/p2 ?/o2)))))"
+ , " (assign ((?test <http://localhost/t2>))"
+ , " (conditional"
+ , " (bgp (triple <http://localhost/t2> ?p1 ?o1))"
+ , " (project (?s1)"
+ , " (bgp (triple ?s1 ?/p2 ?/o2))))))"
+ ) ;
+ check(qs, ops) ;
+ }
+
+ @Test public void test_OptEqualitySubQuery_02() {
+ String qs = StrUtils.strjoinNL
+ ( "SELECT *",
+ "WHERE {",
+ " ?test ?p1 ?o1.",
+ " OPTIONAL {",
+ " SELECT ?test",
+ " { ?s1 ?p2 ?o2 }", //" GROUP BY ?s1",
+ " }" ,
+ "}" ) ;
+ String ops = StrUtils.strjoinNL
+ ( "(disjunction"
+ , " (assign ((?test <http://localhost/t1>))"
+ , " (conditional"
+ , " (bgp (triple <http://localhost/t1> ?p1 ?o1))"
+ , " (project (?s1)"
+ , " (bgp (triple ?s1 ?/p2 ?/o2)))))"
+ , " (assign ((?test <http://localhost/t2>))"
+ , " (conditional"
+ , " (bgp (triple <http://localhost/t2> ?p1 ?o1))"
+ , " (project (?s1)"
+ , " (bgp (triple ?s1 ?/p2 ?/o2))))))"
+ ) ;
+
+ check(qs, ops) ;
+ }
+ @Test public void test_OptEqualitySubQuery_03() {
+ String qs = StrUtils.strjoinNL
+ ( "SELECT *"
+ , "WHERE {"
+ , " ?test ?p1 ?X."
+ , " FILTER ( ?test = <http://localhost/t1> )"
+ , " { SELECT ?s1 ?test { ?test ?p2 ?o2 } }"
+ , "}") ;
+
+ String ops = StrUtils.strjoinNL
+ ( "(disjunction"
+ , " (assign ((?test <http://localhost/t1>))"
+ , " (conditional"
+ , " (bgp (triple <http://localhost/t1> ?p1 ?o1))"
+ , " (project (?s1)"
+ , " (bgp (triple ?s1 ?/p2 ?/o2)))))"
+ , " (assign ((?test <http://localhost/t2>))"
+ , " (conditional"
+ , " (bgp (triple <http://localhost/t2> ?p1 ?o1))"
+ , " (project (?s1)"
+ , " (bgp (triple ?s1 ?/p2 ?/o2))))))"
+ ) ;
+ check(qs, ops) ;
+ }
+ @Test public void test_OptEqualitySubQuery_04() {
+ String qs = StrUtils.strjoinNL
+ ( "SELECT *"
+ , "WHERE {"
+ , " ?test ?p1 ?X."
+ , " FILTER ( ?test = <http://localhost/t1> )"
+ , " { SELECT ?s1 { ?test ?p2 ?o2 } }"
+ , "}") ;
+ String ops = StrUtils.strjoinNL
+ ( "(disjunction"
+ , " (assign ((?test <http://localhost/t1>))"
+ , " (conditional"
+ , " (bgp (triple <http://localhost/t1> ?p1 ?o1))"
+ , " (project (?s1)"
+ , " (bgp (triple ?s1 ?/p2 ?/o2)))))"
+ , " (assign ((?test <http://localhost/t2>))"
+ , " (conditional"
+ , " (bgp (triple <http://localhost/t2> ?p1 ?o1))"
+ , " (project (?s1)"
+ , " (bgp (triple ?s1 ?/p2 ?/o2))))))"
+ ) ;
+
+ check(qs, ops) ;
+ }
+
+ // JENA-383
+ @Test public void test_OptEqualitySubQuery_05() {
+ String qs = StrUtils.strjoinNL
+ ( "PREFIX ex: <http://example.org/test#>"
+ , "SELECT * {"
+ , " ?var ex:p1 ?x."
+ , " OPTIONAL {"
+ , " ?x ex:p2 ?y."
+ , " OPTIONAL { ?y ex:p3 ?z }"
+ , " }"
+ , " FILTER (?var = ex:i)"
+ , " }"
+ ) ;
+ String ops = StrUtils.strjoinNL
+ ( "(filter (= ?var <http://example.org/test#i>)"
+ , " (conditional"
+ , " (bgp (triple ?var <http://example.org/test#p1> ?x))"
+ , " (conditional"
+ , " (bgp (triple ?x <http://example.org/test#p2> ?y))"
+ , " (bgp (triple ?y <http://example.org/test#p3> ?z)))"
+ , " ))"
+ ) ;
+ check(qs, ops) ;
+ }
+
+
+
+ private void check(String qs, String ops)
+ {
+ TestOptimizer.check(qs, ops) ;
+
+// //String queryString = "PREFIX : <http://example/>\n"+qs ;
+// String queryString = qs ;
+// Query query = QueryFactory.create(queryString) ;
+// Op opQuery = Algebra.compile(query) ;
+// Op opOptimize = Algebra.optimize(opQuery) ;
+// Op opExpected = SSE.parseOp(ops) ;
+//
+// System.out.println(query) ;
+// System.out.print(opOptimize) ;
+// System.out.print(opExpected) ;
+// System.out.println("- - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - - - - -") ;
+ }
+
+// String qs1 = StrUtils.strjoinNL
+// ( "SELECT *",
+// "WHERE {",
+// " ?test ?p1 ?o1.",
+// " FILTER ( ?test = <http://localhost/t1> || ?test =
<http://localhost/t2> )",
+// //" FILTER ( ?test = <http://localhost/t1> )",
+// " OPTIONAL {",
+// " SELECT ?s1", // Not ?test
+// " { ?s1 ?p2 ?o2 }", //" GROUP BY ?s1",
+// " }" ,
+// "}" ) ;
+//
+//// No (conditional) optimization.
+//String qs2 = StrUtils.strjoinNL
+// ( "SELECT *",
+// "WHERE {",
+// " ?test ?p1 ?o1.",
+// " OPTIONAL {",
+// " SELECT ?test",
+// " { ?s1 ?p2 ?o2 }", //" GROUP BY ?s1",
+// " }" ,
+// "}" ) ;
+//
+//// ?test in SELECT can't be simply substituted.
+//String qs3 = StrUtils.strjoinNL
+// ( "SELECT *"
+// , "WHERE {"
+// , " ?test ?p1 ?X."
+// , " FILTER ( ?test = <http://localhost/t1> )"
+// , " { SELECT ?s1 ?test { ?test ?p2 ?o2 } }"
+// , "}") ;
+//
+//String qs4 = StrUtils.strjoinNL
+// ( "SELECT *"
+// , "WHERE {"
+// , " ?test ?p1 ?X."
+// , " FILTER ( ?test = <http://localhost/t1> )"
+// , " { SELECT ?s1 { ?test ?p2 ?o2 } }"
+// , "}") ;
+//
+
+
}
Modified: jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Run.java
URL:
http://svn.apache.org/viewvc/jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Run.java?rev=1467806&r1=1467805&r2=1467806&view=diff
==============================================================================
--- jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Run.java (original)
+++ jena/Scratch/AFS/Jena-Dev/trunk/src/dev/Run.java Sun Apr 14 17:59:16 2013
@@ -26,6 +26,7 @@ import com.hp.hpl.jena.query.QueryFactor
import com.hp.hpl.jena.sparql.algebra.Algebra ;
import com.hp.hpl.jena.sparql.algebra.Op ;
import com.hp.hpl.jena.sparql.algebra.optimize.Optimize ;
+import com.hp.hpl.jena.sparql.algebra.optimize.TransformFilterEquality ;
import com.hp.hpl.jena.sparql.serializer.SerializationContext ;
import com.hp.hpl.jena.sparql.sse.writers.WriterOp ;
@@ -77,7 +78,7 @@ public class Run extends RunBase
WriterOp.output(System.out, op, sCxt) ;
- Op op2 = Optimize.apply(new TransformFilterEquality2(), op) ;
+ Op op2 = Optimize.apply(new TransformFilterEquality(), op) ;
System.out.println() ;
WriterOp.output(System.out, op2, sCxt) ;
//System.out.println(op2) ;
Modified:
jena/Scratch/AFS/Jena-Dev/trunk/src/reports/archive/Jena154_GraphVarScope.java
URL:
http://svn.apache.org/viewvc/jena/Scratch/AFS/Jena-Dev/trunk/src/reports/archive/Jena154_GraphVarScope.java?rev=1467806&r1=1467805&r2=1467806&view=diff
==============================================================================
---
jena/Scratch/AFS/Jena-Dev/trunk/src/reports/archive/Jena154_GraphVarScope.java
(original)
+++
jena/Scratch/AFS/Jena-Dev/trunk/src/reports/archive/Jena154_GraphVarScope.java
Sun Apr 14 17:59:16 2013
@@ -47,10 +47,7 @@ public class Jena154_GraphVarScope
//"SELECT ?g WHERE { GRAPH ?g { ?s ?p ?o . } }";
public static void main(String[] args) {
- Op op1 = SSE.parseOp("(quadpattern (quad ?g ?s ?p ?o))") ;
- Op op2 = SSE.parseOp("(quadpattern (quad ?g ?s ?p ?o))") ;
-
- System.out.println(op1.equals(op2)) ;
+ print() ;
//execute() ;
}
@@ -59,21 +56,21 @@ public class Jena154_GraphVarScope
{
String tests[] = {
null
- , "{ GRAPH ?g { ?s ?p ?o } }"
- // Not nested
- , "{ GRAPH ?g { ?s ?p ?o } GRAPH ?g1 { ?s1 ?p1 ?o1 } }"
- , "{ GRAPH ?g { ?s ?p ?o } GRAPH ?g { ?s1 ?p1 ?o1 } }"
- // Nested
- , "{ GRAPH ?g { ?s ?p ?o GRAPH ?g1 { ?s1 ?p1 ?o1 } } }"
- , "{ GRAPH ?g { ?s ?p ?o GRAPH ?g { ?s1 ?p1 ?o1 } } }"
+// , "{ GRAPH ?g { ?s ?p ?o } }"
+// // Not nested
+// , "{ GRAPH ?g { ?s ?p ?o } GRAPH ?g1 { ?s1 ?p1 ?o1 } }"
+// , "{ GRAPH ?g { ?s ?p ?o } GRAPH ?g { ?s1 ?p1 ?o1 } }"
+// // Nested
+// , "{ GRAPH ?g { ?s ?p ?o GRAPH ?g1 { ?s1 ?p1 ?o1 } } }"
+// , "{ GRAPH ?g { ?s ?p ?o GRAPH ?g { ?s1 ?p1 ?o1 } } }"
// Filters
, "{ GRAPH ?g { ?s ?p ?o FILTER (str(?g) = 'graphURI') } }"
- , "{ GRAPH ?g { ?s ?p ?o GRAPH ?g1 { ?s1 ?p1 ?o1 FILTER (str(?g) =
'graphURI') } } }"
- , "{ GRAPH ?g { ?s ?p ?o GRAPH ?g1 { ?s1 ?p1 ?o1 FILTER (str(?g1)
= 'graphURI') } } }"
- // Tricky pattern ... twice.
- , "{ GRAPH ?g { ?s ?p ?o FILTER (str(?g) = 'graphURI') } " +
- " GRAPH ?g { ?s ?p ?o FILTER (str(?g) = 'graphURI') }"
- + "}"
+// , "{ GRAPH ?g { ?s ?p ?o GRAPH ?g1 { ?s1 ?p1 ?o1 FILTER (str(?g)
= 'graphURI') } } }"
+// , "{ GRAPH ?g { ?s ?p ?o GRAPH ?g1 { ?s1 ?p1 ?o1 FILTER
(str(?g1) = 'graphURI') } } }"
+// // Tricky pattern ... twice.
+// , "{ GRAPH ?g { ?s ?p ?o FILTER (str(?g) = 'graphURI') } " +
+// " GRAPH ?g { ?s ?p ?o FILTER (str(?g) = 'graphURI') }"
+// + "}"
} ;
@@ -168,7 +165,7 @@ public class Jena154_GraphVarScope
{
DatasetGraph dsg = DatasetGraphFactory.createMem() ;
setup(dsg) ;
- List<String> vars = Var.varNames(OpVars.patternVars(op)) ;
+ List<String> vars = Var.varNames(OpVars.visibleVars(op)) ;
System.out.println(op) ;
QueryIterator qIter = Algebra.execRef(op, dsg) ;
ResultSet results = ResultSetFactory.create(qIter, vars) ;
