Author: andy
Date: Tue Nov 19 21:28:35 2013
New Revision: 1543581
URL: http://svn.apache.org/r1543581
Log:
Clean up after turning JENA-595 work into a patch file.
Removed:
jena/Scratch/AFS/Dev/src-dev/opt/Jena383_BadPlanOptionals.java
jena/Scratch/AFS/Dev/src-dev/opt/JenaNNN_FilterEquality.java
jena/Scratch/AFS/Dev/src-dev/opt/OpVarsFixed.java
jena/Scratch/AFS/Dev/src-dev/opt/TestFilterPlacement.java
jena/Scratch/AFS/Dev/src-dev/opt/TransformFilterPlacement_Rewrite.java
jena/Scratch/AFS/Dev/src-dev/opt/VarFinder2.java
Modified:
jena/Scratch/AFS/Dev/src-dev/opt/OptMain.java
Modified: jena/Scratch/AFS/Dev/src-dev/opt/OptMain.java
URL:
http://svn.apache.org/viewvc/jena/Scratch/AFS/Dev/src-dev/opt/OptMain.java?rev=1543581&r1=1543580&r2=1543581&view=diff
==============================================================================
--- jena/Scratch/AFS/Dev/src-dev/opt/OptMain.java (original)
+++ jena/Scratch/AFS/Dev/src-dev/opt/OptMain.java Tue Nov 19 21:28:35 2013
@@ -18,32 +18,21 @@
package opt;
-import static com.hp.hpl.jena.sparql.algebra.optimize.Optimize.apply ;
-
-import java.util.Set ;
-
import org.apache.jena.atlas.lib.StrUtils ;
-import com.hp.hpl.jena.query.ARQ ;
import com.hp.hpl.jena.query.Query ;
import com.hp.hpl.jena.query.QueryFactory ;
-import com.hp.hpl.jena.sparql.ARQConstants ;
-import com.hp.hpl.jena.sparql.algebra.* ;
-import com.hp.hpl.jena.sparql.algebra.op.OpJoin ;
-import com.hp.hpl.jena.sparql.algebra.op.OpLabel ;
-import com.hp.hpl.jena.sparql.algebra.optimize.* ;
-import com.hp.hpl.jena.sparql.algebra.optimize.Optimize.RewriterFactory ;
-import com.hp.hpl.jena.sparql.core.Var ;
-import com.hp.hpl.jena.sparql.engine.main.JoinClassifier ;
+import com.hp.hpl.jena.sparql.algebra.Algebra ;
+import com.hp.hpl.jena.sparql.algebra.Op ;
+import com.hp.hpl.jena.sparql.algebra.Transform ;
+import com.hp.hpl.jena.sparql.algebra.Transformer ;
+import com.hp.hpl.jena.sparql.algebra.optimize.TransformFilterPlacement ;
import com.hp.hpl.jena.sparql.sse.SSE ;
-import com.hp.hpl.jena.sparql.util.Context ;
public class OptMain {
- // LeftJoin - Fixed vars only (LHS) OpVars. VarFinder.fixed(null)
- // UNION - fixed : intersection
- // Test - UNION
-
- // Also TransformDistinctToReduced (JENA-585)and
TransformOrderByDistincApplication
+
+
+ // Also TransformDistinctToReduced (JENA-585) and
TransformOrderByDistincApplication
// JENA-383 : The query optimizer generates a suboptimal query plan in
case of nested optionals followed by a filter
@@ -54,125 +43,21 @@ public class OptMain {
// Was Op.equals a good idea?
// Calculate vars for an expr once and cache : op.getVars.
- // Checking:
- // filter-equality-13 -- scope issue for leftjoin, conditional in a
sequence.
- // Can't push in the filer to the sequence as may yield no binding ->
need fixed vars
- // TestTranformFilters :
- // optionalEqualitySubQuery_01 -- looks OK -- JENA-432
- // optionalEquality_01 JENA-383 -- WRONG - Lost RHS of conditional
- // optionalEqualityScope_01 -- WRONG -
-
- // Double push in a sequence of conditional then BGP.
- // JoinStartegy and FilterPlacement interaction.
-
public static void main(String... argv) {
- fixedVars() ;
placement() ;
}
- public static void fixedVars() {
- //. FILTERs
- // See alt VarFinder
-// String sx = StrUtils.strjoinNL
-// ("(union",
-// " (leftJoin (filter (= ?x 123) (bgp (?s ?p ?o1))) (bgp (?s ?p
?o2)) )",
-// " (leftJoin (bgp (?s ?p2 ?o1)) (bgp (?x ?p1 ?o2)) )",
-// ")"
-// ) ;
- String sx = "(filter (= ?x 123) (extend (?x 567) (bgp (?s ?p ?o1)) ))"
;
- //Op op = SSE.parseOp("(union (bgp (?s ?p ?o1)) (bgp (?s ?p ?o2)))") ;
- Op op = SSE.parseOp(sx) ;
- System.out.println(op);
- Set<Var> x = OpVarsFixed.fixed(op) ;
- System.out.println(x) ;
- System.exit(0) ;
- }
-
- public static void testFailure() {
- String DIR = "/home/afs/Jena/jena-arq/testing/ARQ/OptFilterEquality" ;
-
- arq.sparql.main(
"--query="+DIR+"/filter-equality-13.rq", "--data="+DIR+"/data-4.ttl");
- rewire() ;
- arq.sparql.main(
"--query="+DIR+"/filter-equality-13.rq", "--data="+DIR+"/data-4.ttl");
- dewire() ;
-// arq.sparql.main(
"--query="+DIR+"/filter-equality-13.rq", "--data="+DIR+"/data-4.ttl");
-// System.exit(0) ;
-
- Query q = QueryFactory.read(DIR+"/filter-equality-13.rq") ;
- Op op1 = Algebra.compile(q) ;
- Op op2 = Algebra.optimize(op1) ;
- System.out.println(op2);
- rewire() ;
- // Pushing filter over the (conditional) is wrong
- // Need to look for MUST be fixed variables, which is not what OpVars.
mentioned vars is.
- // nor is VarFinder.fixed (but is that wrong?)
-
-/*
-(filter (= ?x <http://example.org/x1>)
- (sequence
- (conditional
- (table unit)
- (bgp (triple ?x <http://example.org/qq> ?o2)))
- (bgp (triple ?x <http://example.org/p> ?o1))))
-
-Rewrite2
-(sequence
- (filter (= ?x <http://example.org/x1>)
- (conditional
- (table unit)
- (bgp (triple ?x <http://example.org/qq> ?o2))))
- (filter (= ?x <http://example.org/x1>)
- (bgp (triple ?x <http://example.org/p> ?o1))))
-*/
- op2 = Algebra.optimize(op1) ;
- System.out.println(op2);
-
- System.exit(0) ;
- }
-
- static void varFinder() {
- if ( false )
- {
- Op op = SSE.parseOp("(union (bgp (?s ?p ?o1)) (bgp (?s ?p ?o2)))")
;
- System.out.println(op) ;
- // ** WRONG?? Exact meaning of "defined"?
- System.out.println(VarFinder2.fixed(op)) ;
- System.out.println(VarFinder2.optDefined(op)) ;
- System.exit(0) ;
- }
- {
- // See alt VarFinder
- String[] x = {
- "(join (bgp (?s ?p ?o)) (union (bgp (?s ?p ?o)) (bgp (?s ?p
?o1))) )",
- "(join (bgp (?s ?p ?o)) (union (bgp (?s ?p ?z)) (bgp (?s ?p1
?z))) )"
- } ;
-
- for (String s : x ) {
- System.out.println("--------") ;
- Op op = SSE.parseOp(s) ;
- System.out.println(op) ;
- System.out.println("fixed: "+VarFinder2.fixed(op)) ;
- System.out.println("Opt: "+VarFinder2.optDefined(op)) ;
- boolean nonLinear = JoinClassifier.isLinear((OpJoin)op) ;
- System.out.println(nonLinear) ;
- }
- System.exit(0) ;
- }
- }
-
static void placement() {
if ( false ) {
//String input = "(filter (?x 123) (sequence (conditional (table
unit)
String input = StrUtils.strjoinNL
("(filter (= ?x <http://example/x>)",
" (sequence",
- " (conditional",
- " (table unit)",
- " (bgp (triple ?x <http://example/q> ?o)))",
+ " (bgp (triple ?x <http://example/p> ?o2))",
" (bgp (triple ?x <http://example/p> ?o2))))"
) ;
- Transform t_placement = new TransformFilterPlacement_Rewrite() ;
+ Transform t_placement = new TransformFilterPlacement() ;
Op op1 = SSE.parseOp(input) ;
Op op2 = op1 ;
op2 = Algebra.optimize(op1) ;
@@ -219,30 +104,20 @@ SELECT *
, " ?x :p ?o2"
, "}"
) ;
-
+ String qs4 = StrUtils.strjoinNL
+ ("SELECT *"
+ , "WHERE {"
+ , " ?test ?p1 ?X."
+ , " FILTER ( ?test = <http://localhost/t1> )"
+ , " { SELECT ?s1 ?test { ?test ?p2 ?o2 } }"
+ , "}") ;
- Query query = QueryFactory.create(qs3) ;
+ Query query = QueryFactory.create(qs4) ;
Op op1 = Algebra.compile(query) ;
System.out.println("** Input") ;
System.out.println(op1) ;
- {
- System.out.println("** OLD") ;
- Op op2 = Algebra.optimize(op1) ;
- System.out.println(op2) ;
- }
- System.out.println("** NEW : Place/Join") ;
- rewire() ;
- //Op op2 = new Rewrite2(ARQ.getContext()).rewrite(op1) ;
- Op op2 = op1 ;
- op2 = Transformer.transform(new TransformJoinStrategy(), op2) ;
- op2 = placement(op2) ;
- System.out.println(op2) ;
- System.out.println("** NEW : Join/Place") ;
-
- op2 = op1 ;
- op2 = placement(op2) ;
- op2 = Transformer.transform(new TransformJoinStrategy(), op2) ;
-
+ System.out.println("** OPTIMIZE") ;
+ Op op2 = Algebra.optimize(op1) ;
System.out.println(op2) ;
System.exit(0) ;
}
@@ -250,138 +125,7 @@ SELECT *
}
static Op placement(Op op) {
- return Transformer.transform(new TransformFilterPlacement_Rewrite(),
op) ;
- }
-
- private static void rewire() {
- Optimize.setFactory(new RewriterFactory2());
- }
-
- private static void dewire() {
- Optimize.setFactory(Optimize.stdOptimizationFactory) ;
- }
-
- static class RewriterFactory2 implements RewriterFactory {
-
- @Override
- public Rewrite create(Context context) {
- return new Rewrite2(context) ;
- }
- }
-
- static class Rewrite2 implements Rewrite {
-
- private Context context ;
-
- public Rewrite2(Context context) {
- this.context = context ;
- }
-
- @Override
- public Op rewrite(Op op)
- {
- System.out.println("Rewrite2") ;
- Context context = ARQ.getContext() ;
- // Record optimizer
- if ( context.get(ARQConstants.sysOptimizer) == null )
- context.set(ARQConstants.sysOptimizer, this) ;
-
- if ( false )
- {
- // Removal of "group of one" join (AKA SPARQL
"simplification")
- // is done during algebra generation in AlgebraGenerator
- op = apply("Simplify", new TransformSimplify(), op) ;
- op = apply("Delabel", new TransformRemoveLabels(), op) ;
- }
-
- // ** TransformScopeRename::
- // This is a requirement for the linearization execution that the
default
- // ARQ query engine uses where possible.
- // This transformation must be done (e.g. by QueryEngineBase) if
no other optimization is done.
- op = TransformScopeRename.transform(op) ;
-
- // Prepare expressions.
- OpWalker.walk(op, new OpVisitorExprPrepare(context)) ;
-
- // Need to allow subsystems to play with this list.
-
- if ( context.isTrueOrUndef(ARQ.propertyFunctions) )
- op = apply("Property Functions", new
TransformPropertyFunction(context), op) ;
-
- if ( context.isTrueOrUndef(ARQ.optFilterConjunction) )
- op = apply("filter conjunctions to ExprLists", new
TransformFilterConjunction(), op) ;
-
- if ( context.isTrueOrUndef(ARQ.optFilterExpandOneOf) )
- op = apply("Break up IN and NOT IN", new
TransformExpandOneOf(), op) ;
-
- // Either, do filter placement and other sequence generating
transformations.
- // or improve to place in a sequence (latter is better?)
-
- if ( context.isTrueOrUndef(ARQ.optFilterImplicitJoin) )
- op = apply("Filter Implicit Join", new
TransformFilterImplicitJoin(), op);
-
- if ( context.isTrueOrUndef(ARQ.optImplicitLeftJoin) )
- op = apply("Implicit Left Join", new
TransformImplicitLeftJoin(), op);
-
- if ( context.isTrueOrUndef(ARQ.optFilterEquality) )
- {
- //boolean termStrings = context.isDefined(ARQ.optTermStrings) ;
- op = apply("Filter Equality", new TransformFilterEquality(),
op) ;
- }
-
- // Can promote table empty at this point since only the implicit
join optimizations
- // are currently capable of introducing it
- if ( context.isTrueOrUndef(ARQ.optPromoteTableEmpty) )
- op = apply("Table Empty Promotion", new
TransformPromoteTableEmpty(), op) ;
-
- if ( context.isTrueOrUndef(ARQ.optFilterDisjunction) )
- op = apply("Filter Disjunction", new
TransformFilterDisjunction(), op) ;
-
- // ***** After JoinStrategy (or don't push left and right in join).
- if ( context.isTrueOrUndef(ARQ.optFilterPlacement) )
- // *****
- op = apply("Filter Placement", new
TransformFilterPlacement_Rewrite(), op) ;
-
-
- if ( context.isTrueOrUndef(ARQ.optTopNSorting) )
- op = apply("TopN Sorting", new TransformTopN(), op) ;
-
- // ORDER BY+DISTINCT optimizations
- // We apply the one that changes evaluation order first since when
it does apply it will give much
- // better performance than just transforming DISTINCT to REDUCED
-
- if ( context.isTrueOrUndef(ARQ.optOrderByDistinctApplication) )
- op = apply("Apply DISTINCT prior to ORDER BY where possible",
new TransformOrderByDistinctAppplication(), op);
-
- // Transform some DISTINCT to REDUCED, slightly more liberal
transform that ORDER BY+DISTINCT application
- // but doesn't improve performance as much though should keep
memory usage down
- if ( context.isTrueOrUndef(ARQ.optDistinctToReduced) )
- op = apply("Distinct replaced with reduced", new
TransformDistinctToReduced(), op) ;
-
- // Convert paths to triple patterns.
- // Also done in the AlgebraGenerator so this transform step
catches programmatically built op expressions
- op = apply("Path flattening", new TransformPathFlattern(), op) ;
-
- // Find joins/leftJoin that can be done by index joins (generally
preferred as fixed memory overhead).
- if ( context.isTrueOrUndef(ARQ.optIndexJoinStrategy) )
- op = apply("Index Join strategy", new TransformJoinStrategy(),
op) ;
-
-// // ***** After JoinStrategy (or don't push left and right in
join).
-// if ( context.isTrueOrUndef(ARQ.optFilterPlacement) )
-// // *****
-// op = apply("Filter Placement", new
TransformFilterPlacement_Rewrite(), op) ;
-
- // Merge adjacent BGPs
- if ( context.isTrueOrUndef(ARQ.optMergeBGPs) )
- op = apply("Merge BGPs", new TransformMergeBGPs(), op) ;
-
- // Mark
- if ( false )
- op = OpLabel.create("Transformed", op) ;
- return op ;
- }
-
-
+ return Transformer.transform(new TransformFilterPlacement(), op) ;
}
}
