PHOENIX-4437 Make QueryPlan.getEstimatedBytesToScan() independent of 
getExplainPlan() and pull optimize() out of getExplainPlan()


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7ef96fe1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7ef96fe1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7ef96fe1

Branch: refs/heads/4.x-HBase-0.98
Commit: 7ef96fe1bed43f3ac3dae900a3e6a83791faf697
Parents: 977699a
Author: maryannxue <maryann....@gmail.com>
Authored: Thu Dec 21 10:31:04 2017 -0800
Committer: maryannxue <maryann....@gmail.com>
Committed: Tue Mar 13 20:08:24 2018 -0700

----------------------------------------------------------------------
 .../end2end/ExplainPlanWithStatsEnabledIT.java  |  4 +-
 .../apache/phoenix/execute/BaseQueryPlan.java   | 45 ++++++--------
 .../apache/phoenix/execute/HashJoinPlan.java    | 59 +++++++++---------
 .../phoenix/execute/SortMergeJoinPlan.java      | 63 ++++++++++----------
 .../org/apache/phoenix/execute/UnionPlan.java   | 53 ++++++++--------
 .../apache/phoenix/jdbc/PhoenixStatement.java   |  9 ++-
 6 files changed, 120 insertions(+), 113 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
index 76ed7ba..a835e84 100644
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
@@ -303,8 +303,8 @@ public class ExplainPlanWithStatsEnabledIT extends 
ParallelStatsEnabledIT {
         try (Connection conn = DriverManager.getConnection(getUrl())) {
             conn.setAutoCommit(false);
             Estimate info = getByteRowEstimates(conn, sql, binds);
-            assertEquals((Long) 200L, info.estimatedBytes);
-            assertEquals((Long) 2L, info.estimatedRows);
+            assertEquals((Long) 176l, info.estimatedBytes);
+            assertEquals((Long) 2l, info.estimatedRows);
             assertTrue(info.estimateInfoTs > 0);
         }
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java 
b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index 38ed926..c4edf31 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -118,7 +118,7 @@ public abstract class BaseQueryPlan implements QueryPlan {
     protected Long estimatedRows;
     protected Long estimatedSize;
     protected Long estimateInfoTimestamp;
-    private boolean explainPlanCalled;
+    private boolean getEstimatesCalled;
     
 
     protected BaseQueryPlan(
@@ -500,32 +500,17 @@ public abstract class BaseQueryPlan implements QueryPlan {
 
     @Override
     public ExplainPlan getExplainPlan() throws SQLException {
-        explainPlanCalled = true;
         if (context.getScanRanges() == ScanRanges.NOTHING) {
             return new ExplainPlan(Collections.singletonList("DEGENERATE SCAN 
OVER " + getTableRef().getTable().getName().getString()));
         }
 
-        // If cost-based optimizer is enabled, we need to initialize a dummy 
iterator to
-        // get the stats for computing costs.
-        boolean costBased =
-                
context.getConnection().getQueryServices().getConfiguration().getBoolean(
-                        QueryServices.COST_BASED_OPTIMIZER_ENABLED, 
QueryServicesOptions.DEFAULT_COST_BASED_OPTIMIZER_ENABLED);
-        if (costBased) {
-            ResultIterator iterator = iterator();
-            iterator.close();
-        }
-        // Optimize here when getting explain plan, as queries don't get 
optimized until after compilation
-        QueryPlan plan = 
context.getConnection().getQueryServices().getOptimizer().optimize(context.getStatement(),
 this);
-        ExplainPlan exp = plan instanceof BaseQueryPlan ? new 
ExplainPlan(getPlanSteps(plan.iterator())) : plan.getExplainPlan();
-        if (!costBased) { // do not override estimates if they are used for 
cost calculation.
-            this.estimatedRows = plan.getEstimatedRowsToScan();
-            this.estimatedSize = plan.getEstimatedBytesToScan();
-            this.estimateInfoTimestamp = plan.getEstimateInfoTimestamp();
-        }
-        return exp;
+        ResultIterator iterator = iterator();
+        ExplainPlan explainPlan = new ExplainPlan(getPlanSteps(iterator));
+        iterator.close();
+        return explainPlan;
     }
 
-    private List<String> getPlanSteps(ResultIterator iterator){
+    private List<String> getPlanSteps(ResultIterator iterator) {
         List<String> planSteps = Lists.newArrayListWithExpectedSize(5);
         iterator.explain(planSteps);
         return planSteps;
@@ -538,26 +523,32 @@ public abstract class BaseQueryPlan implements QueryPlan {
     
     @Override
     public Long getEstimatedRowsToScan() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimatedRows;
     }
 
     @Override
     public Long getEstimatedBytesToScan() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimatedSize;
     }
 
     @Override
     public Long getEstimateInfoTimestamp() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimateInfoTimestamp;
     }
 
+    private void getEstimates() throws SQLException {
+        getEstimatesCalled = true;
+        // Initialize a dummy iterator to get the estimates based on stats.
+        ResultIterator iterator = iterator();
+        iterator.close();
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java 
b/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
index 2d2ff4e..23a0da6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
@@ -99,7 +99,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
     private Long estimatedRows;
     private Long estimatedBytes;
     private Long estimateInfoTs;
-    private boolean explainPlanCalled;
+    private boolean getEstimatesCalled;
     
     public static HashJoinPlan create(SelectStatement statement, 
             QueryPlan plan, HashJoinInfo joinInfo, SubPlan[] subPlans) throws 
SQLException {
@@ -247,7 +247,6 @@ public class HashJoinPlan extends DelegateQueryPlan {
 
     @Override
     public ExplainPlan getExplainPlan() throws SQLException {
-        explainPlanCalled = true;
         List<String> planSteps = 
Lists.newArrayList(delegate.getExplainPlan().getPlanSteps());
         int count = subPlans.length;
         for (int i = 0; i < count; i++) {
@@ -263,26 +262,6 @@ public class HashJoinPlan extends DelegateQueryPlan {
         if (joinInfo != null && joinInfo.getLimit() != null) {
             planSteps.add("    JOIN-SCANNER " + joinInfo.getLimit() + " ROW 
LIMIT");
         }
-        for (SubPlan subPlan : subPlans) {
-            if (subPlan.getInnerPlan().getEstimatedBytesToScan() == null
-                    || subPlan.getInnerPlan().getEstimatedRowsToScan() == null
-                    || subPlan.getInnerPlan().getEstimateInfoTimestamp() == 
null) {
-                /*
-                 * If any of the sub plans doesn't have the estimate info 
available, then we don't
-                 * provide estimate for the overall plan
-                 */
-                estimatedBytes = null;
-                estimatedRows = null;
-                estimateInfoTs = null;
-                break;
-            } else {
-                estimatedBytes =
-                        add(estimatedBytes, 
subPlan.getInnerPlan().getEstimatedBytesToScan());
-                estimatedRows = add(estimatedRows, 
subPlan.getInnerPlan().getEstimatedRowsToScan());
-                estimateInfoTs =
-                        getMin(estimateInfoTs, 
subPlan.getInnerPlan().getEstimateInfoTimestamp());
-            }
-        }
         return new ExplainPlan(planSteps);
     }
 
@@ -520,27 +499,51 @@ public class HashJoinPlan extends DelegateQueryPlan {
 
     @Override
     public Long getEstimatedRowsToScan() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimatedRows;
     }
 
     @Override
     public Long getEstimatedBytesToScan() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimatedBytes;
     }
 
     @Override
     public Long getEstimateInfoTimestamp() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimateInfoTs;
     }
+
+    private void getEstimates() throws SQLException {
+        getEstimatesCalled = true;
+        for (SubPlan subPlan : subPlans) {
+            if (subPlan.getInnerPlan().getEstimatedBytesToScan() == null
+                    || subPlan.getInnerPlan().getEstimatedRowsToScan() == null
+                    || subPlan.getInnerPlan().getEstimateInfoTimestamp() == 
null) {
+                /*
+                 * If any of the sub plans doesn't have the estimate info 
available, then we don't
+                 * provide estimate for the overall plan
+                 */
+                estimatedBytes = null;
+                estimatedRows = null;
+                estimateInfoTs = null;
+                break;
+            } else {
+                estimatedBytes =
+                        add(estimatedBytes, 
subPlan.getInnerPlan().getEstimatedBytesToScan());
+                estimatedRows = add(estimatedRows, 
subPlan.getInnerPlan().getEstimatedRowsToScan());
+                estimateInfoTs =
+                        getMin(estimateInfoTs, 
subPlan.getInnerPlan().getEstimateInfoTimestamp());
+            }
+        }
+    }
 }
 
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java 
b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
index 3e380da..2436d1e 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
@@ -95,7 +95,7 @@ public class SortMergeJoinPlan implements QueryPlan {
     private Long estimatedBytes;
     private Long estimatedRows;
     private Long estimateInfoTs;
-    private boolean explainPlanCalled;
+    private boolean getEstimatesCalled;
 
     public SortMergeJoinPlan(StatementContext context, FilterableStatement 
statement, TableRef table, 
             JoinType type, QueryPlan lhsPlan, QueryPlan rhsPlan, 
List<Expression> lhsKeyExpressions, List<Expression> rhsKeyExpressions,
@@ -157,7 +157,6 @@ public class SortMergeJoinPlan implements QueryPlan {
 
     @Override
     public ExplainPlan getExplainPlan() throws SQLException {
-        explainPlanCalled = true;
         List<String> steps = Lists.newArrayList();
         steps.add("SORT-MERGE-JOIN (" + type.toString().toUpperCase() + ") 
TABLES");
         for (String step : lhsPlan.getExplainPlan().getPlanSteps()) {
@@ -167,28 +166,6 @@ public class SortMergeJoinPlan implements QueryPlan {
         for (String step : rhsPlan.getExplainPlan().getPlanSteps()) {
             steps.add("    " + step);            
         }
-        if ((lhsPlan.getEstimatedBytesToScan() == null || 
rhsPlan.getEstimatedBytesToScan() == null)
-                || (lhsPlan.getEstimatedRowsToScan() == null
-                        || rhsPlan.getEstimatedRowsToScan() == null)
-                || (lhsPlan.getEstimateInfoTimestamp() == null
-                        || rhsPlan.getEstimateInfoTimestamp() == null)) {
-            /*
-             * If any of the sub plans doesn't have the estimate info 
available, then we don't
-             * provide estimate for the overall plan
-             */
-            estimatedBytes = null;
-            estimatedRows = null;
-            estimateInfoTs = null;
-        } else {
-            estimatedBytes =
-                    add(add(estimatedBytes, lhsPlan.getEstimatedBytesToScan()),
-                        rhsPlan.getEstimatedBytesToScan());
-            estimatedRows =
-                    add(add(estimatedRows, lhsPlan.getEstimatedRowsToScan()),
-                        rhsPlan.getEstimatedRowsToScan());
-            estimateInfoTs =
-                    getMin(lhsPlan.getEstimateInfoTimestamp(), 
rhsPlan.getEstimateInfoTimestamp());
-        }
         return new ExplainPlan(steps);
     }
 
@@ -754,25 +731,51 @@ public class SortMergeJoinPlan implements QueryPlan {
 
     @Override
     public Long getEstimatedRowsToScan() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimatedRows;
     }
 
     @Override
     public Long getEstimatedBytesToScan() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimatedBytes;
     }
 
     @Override
     public Long getEstimateInfoTimestamp() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimateInfoTs;
     }
+
+    private void getEstimates() throws SQLException {
+        getEstimatesCalled = true;
+        if ((lhsPlan.getEstimatedBytesToScan() == null || 
rhsPlan.getEstimatedBytesToScan() == null)
+                || (lhsPlan.getEstimatedRowsToScan() == null
+                || rhsPlan.getEstimatedRowsToScan() == null)
+                || (lhsPlan.getEstimateInfoTimestamp() == null
+                || rhsPlan.getEstimateInfoTimestamp() == null)) {
+            /*
+             * If any of the sub plans doesn't have the estimate info 
available, then we don't
+             * provide estimate for the overall plan
+             */
+            estimatedBytes = null;
+            estimatedRows = null;
+            estimateInfoTs = null;
+        } else {
+            estimatedBytes =
+                    add(add(estimatedBytes, lhsPlan.getEstimatedBytesToScan()),
+                            rhsPlan.getEstimatedBytesToScan());
+            estimatedRows =
+                    add(add(estimatedRows, lhsPlan.getEstimatedRowsToScan()),
+                            rhsPlan.getEstimatedRowsToScan());
+            estimateInfoTs =
+                    getMin(lhsPlan.getEstimateInfoTimestamp(), 
rhsPlan.getEstimateInfoTimestamp());
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java 
b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
index e6bf654..3b5168c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
@@ -69,7 +69,7 @@ public class UnionPlan implements QueryPlan {
     private Long estimatedRows;
     private Long estimatedBytes;
     private Long estimateInfoTs;
-    private boolean explainPlanCalled;
+    private boolean getEstimatesCalled;
 
     public UnionPlan(StatementContext context, FilterableStatement statement, 
TableRef table, RowProjector projector,
             Integer limit, Integer offset, OrderBy orderBy, GroupBy groupBy, 
List<QueryPlan> plans, ParameterMetaData paramMetaData) throws SQLException {
@@ -174,7 +174,6 @@ public class UnionPlan implements QueryPlan {
 
     @Override
     public ExplainPlan getExplainPlan() throws SQLException {
-        explainPlanCalled = true;
         List<String> steps = new ArrayList<String>();
         steps.add("UNION ALL OVER " + this.plans.size() + " QUERIES");
         ResultIterator iterator = iterator();
@@ -184,23 +183,6 @@ public class UnionPlan implements QueryPlan {
         for (int i = 1 ; i < steps.size()-offset; i++) {
             steps.set(i, "    " + steps.get(i));
         }
-        for (QueryPlan plan : plans) {
-            if (plan.getEstimatedBytesToScan() == null || 
plan.getEstimatedRowsToScan() == null
-                    || plan.getEstimateInfoTimestamp() == null) {
-                /*
-                 * If any of the sub plans doesn't have the estimate info 
available, then we don't
-                 * provide estimate for the overall plan
-                 */
-                estimatedBytes = null;
-                estimatedRows = null;
-                estimateInfoTs = null;
-                break;
-            } else {
-                estimatedBytes = add(estimatedBytes, 
plan.getEstimatedBytesToScan());
-                estimatedRows = add(estimatedRows, 
plan.getEstimatedRowsToScan());
-                estimateInfoTs = getMin(estimateInfoTs, 
plan.getEstimateInfoTimestamp());
-            }
-        }
         return new ExplainPlan(steps);
     }
 
@@ -265,25 +247,46 @@ public class UnionPlan implements QueryPlan {
 
     @Override
     public Long getEstimatedRowsToScan() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimatedRows;
     }
 
     @Override
     public Long getEstimatedBytesToScan() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimatedBytes;
     }
 
     @Override
     public Long getEstimateInfoTimestamp() throws SQLException {
-        if (!explainPlanCalled) {
-            getExplainPlan();
+        if (!getEstimatesCalled) {
+            getEstimates();
         }
         return estimateInfoTs;
     }
+
+    private void getEstimates() throws SQLException {
+        getEstimatesCalled = true;
+        for (QueryPlan plan : plans) {
+            if (plan.getEstimatedBytesToScan() == null || 
plan.getEstimatedRowsToScan() == null
+                    || plan.getEstimateInfoTimestamp() == null) {
+                /*
+                 * If any of the sub plans doesn't have the estimate info 
available, then we don't
+                 * provide estimate for the overall plan
+                 */
+                estimatedBytes = null;
+                estimatedRows = null;
+                estimateInfoTs = null;
+                break;
+            } else {
+                estimatedBytes = add(estimatedBytes, 
plan.getEstimatedBytesToScan());
+                estimatedRows = add(estimatedRows, 
plan.getEstimatedRowsToScan());
+                estimateInfoTs = getMin(estimateInfoTs, 
plan.getEstimateInfoTimestamp());
+            }
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java 
b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index 94aeb34..da8beae 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -578,7 +578,14 @@ public class PhoenixStatement implements Statement, 
SQLCloseable {
         @Override
         public QueryPlan compilePlan(PhoenixStatement stmt, Sequence.ValueOp 
seqAction) throws SQLException {
             CompilableStatement compilableStmt = getStatement();
-            final StatementPlan plan = compilableStmt.compilePlan(stmt, 
Sequence.ValueOp.VALIDATE_SEQUENCE);
+            StatementPlan compilePlan = compilableStmt.compilePlan(stmt, 
Sequence.ValueOp.VALIDATE_SEQUENCE);
+            // For a QueryPlan, we need to get its optimized plan; for a 
MutationPlan, its enclosed QueryPlan
+            // has already been optimized during compilation.
+            if (compilePlan instanceof QueryPlan) {
+                QueryPlan dataPlan = (QueryPlan) compilePlan;
+                compilePlan = 
stmt.getConnection().getQueryServices().getOptimizer().optimize(stmt, dataPlan);
+            }
+            final StatementPlan plan = compilePlan;
             List<String> planSteps = plan.getExplainPlan().getPlanSteps();
             List<Tuple> tuples = 
Lists.newArrayListWithExpectedSize(planSteps.size());
             Long estimatedBytesToScan = plan.getEstimatedBytesToScan();

Reply via email to