IMPALA-5440: Add planner tests with extreme statistics values This commit address some of the issues in JIRA: tests against the cardinality overflowing from JOIN, UNION, CROSS JOIN, FULL OUTER JOIN, 0 row number and negative row number, as well as cardinality on Subplan node.
Change-Id: I86dec47cf1438882cafaec53e97864ccfcdff3cb Reviewed-on: http://gerrit.cloudera.org:8080/9065 Reviewed-by: Alex Behm <alex.b...@cloudera.com> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/955fca68 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/955fca68 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/955fca68 Branch: refs/heads/master Commit: 955fca68d53499455fceb5d7d308957eb6687a87 Parents: f4d7e26 Author: Xinran Yu Tinney <xyutin...@cloudera.com> Authored: Thu Jan 18 15:27:31 2018 -0600 Committer: Impala Public Jenkins <impala-public-jenk...@gerrit.cloudera.org> Committed: Tue Feb 13 04:26:34 2018 +0000 ---------------------------------------------------------------------- .../org/apache/impala/planner/PlannerTest.java | 77 ++++++++++++++++++++ .../apache/impala/planner/PlannerTestBase.java | 40 ++++++++++ 2 files changed, 117 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/955fca68/fe/src/test/java/org/apache/impala/planner/PlannerTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java index 760334d..5dbba75 100644 --- a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java +++ b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java @@ -504,4 +504,81 @@ public class PlannerTest extends PlannerTestBase { options.setExplain_level(TExplainLevel.EXTENDED); runPlannerTestFile("min-max-runtime-filters", options); } + + @Test + public void testCardinalityOverflow() throws ImpalaException { + String tblName = "tpch.cardinality_overflow"; + String colDefs = "(" + + "l_orderkey BIGINT, " + + "l_partkey BIGINT, " + + "l_suppkey BIGINT, " + + "l_linenumber INT, " + + "l_shipmode STRING, " + + "l_comment STRING" + + ")"; + String tblLocation = "LOCATION " + + "'hdfs://localhost:20500/test-warehouse/tpch.lineitem'"; + String tblPropsTemplate = "TBLPROPERTIES('numRows'='%s')"; + String tblProps = String.format(tblPropsTemplate, Long.toString(Long.MAX_VALUE)); + + addTestTable(String.format("CREATE EXTERNAL TABLE %s %s %s %s;", + tblName, colDefs, tblLocation, tblProps)); + + // CROSS JOIN query: tests that multiplying the input cardinalities does not overflow + // the cross-join's estimated cardinality + String query = "select * from tpch.cardinality_overflow a," + + "tpch.cardinality_overflow b, tpch.cardinality_overflow c"; + checkCardinality(query, 0, Long.MAX_VALUE); + + // FULL OUTER JOIN query: tests that adding the input cardinalities does not overflow + // the full outer join's estimated cardinality + query = "select a.l_comment from tpch.cardinality_overflow a full outer join " + + "tpch.cardinality_overflow b on a.l_orderkey = b.l_partkey"; + checkCardinality(query, 0, Long.MAX_VALUE); + + // UNION query: tests that adding the input cardinalities does not overflow + // the union's estimated cardinality + query = "select l_shipmode from tpch.cardinality_overflow " + + "union select l_comment from tpch.cardinality_overflow"; + checkCardinality(query, 0, Long.MAX_VALUE); + + // JOIN query: tests that multiplying the input cardinalities does not overflow + // the join's estimated cardinality + query = "select a.l_comment from tpch.cardinality_overflow a inner join " + + "tpch.cardinality_overflow b on a.l_linenumber < b.l_orderkey"; + checkCardinality(query, 0, Long.MAX_VALUE); + + // creates an empty table and tests that the cardinality is 0 + tblName = "tpch.ex_customer_cardinality_zero"; + tblProps = String.format(tblPropsTemplate, 0); + addTestTable(String.format("CREATE EXTERNAL TABLE %s %s %s %s;", + tblName, colDefs, tblLocation, tblProps)); + query = "select * from tpch.ex_customer_cardinality_zero"; + checkCardinality(query, 0, 0); + + // creates a table with negative row count and + // tests that the cardinality is not negative + tblName = "tpch.ex_customer_cardinality_neg"; + tblProps = String.format(tblPropsTemplate, -1); + addTestTable(String.format("CREATE EXTERNAL TABLE %s %s %s %s;", + tblName, colDefs, tblLocation, tblProps)); + query = "select * from tpch.ex_customer_cardinality_neg"; + checkCardinality(query, -1, Long.MAX_VALUE); + + // SUBPLAN query: tests that adding the input cardinalities does not overflow + // the SUBPLAN's estimated cardinality + tblName = "functional_parquet.cardinality_overflow"; + colDefs = "(" + + "id BIGINT, " + + "int_array ARRAY<INT>" + + ")"; + String storedAs = "STORED AS PARQUET"; + tblLocation = "LOCATION " + + "'hdfs://localhost:20500/test-warehouse/complextypestbl_parquet'"; + tblProps = String.format(tblPropsTemplate, Long.toString(Long.MAX_VALUE)); + addTestTable(String.format("CREATE EXTERNAL TABLE %s %s %s %s %s;", + tblName, colDefs, storedAs, tblLocation, tblProps)); + query = "select id from functional_parquet.cardinality_overflow t, t.int_array"; + checkCardinality(query, 0, Long.MAX_VALUE); + } } http://git-wip-us.apache.org/repos/asf/impala/blob/955fca68/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java index bc93160..7c47d74 100644 --- a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java +++ b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java @@ -36,6 +36,7 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.fs.Path; import org.apache.impala.analysis.ColumnLineageGraph; import org.apache.impala.analysis.DescriptorTable; +import org.apache.impala.catalog.Catalog; import org.apache.impala.catalog.CatalogException; import org.apache.impala.common.FrontendTestBase; import org.apache.impala.common.ImpalaException; @@ -665,6 +666,45 @@ public class PlannerTestBase extends FrontendTestBase { } } + /** + * This function plans the given query and fails if the estimated cardinalities are + * not within the specified bounds [min, max]. + */ + protected void checkCardinality(String query, long min, long max) + throws ImpalaException { + TQueryCtx queryCtx = TestUtils.createQueryContext(Catalog.DEFAULT_DB, + System.getProperty("user.name")); + queryCtx.client_request.setStmt(query); + StringBuilder explainBuilder = new StringBuilder(); + TExecRequest execRequest = frontend_.createExecRequest(queryCtx, explainBuilder); + + if (!execRequest.isSetQuery_exec_request() + || execRequest.query_exec_request == null + || execRequest.query_exec_request.plan_exec_info == null) { + return; + } + for (TPlanExecInfo execInfo : execRequest.query_exec_request.plan_exec_info) { + for (TPlanFragment planFragment : execInfo.fragments) { + if (!planFragment.isSetPlan() || planFragment.plan == null) continue; + for (TPlanNode node : planFragment.plan.nodes) { + if (node.estimated_stats == null) { + fail("Query: " + query + " has no estimated statistics"); + } + long cardinality = node.estimated_stats.cardinality; + if (cardinality < min || cardinality > max) { + StringBuilder errorLog = new StringBuilder(); + errorLog.append("Query: " + query + "\n"); + errorLog.append( + "Expected cardinality estimate between " + min + " and " + max + "\n"); + errorLog.append("Actual cardinality estimate: " + cardinality + "\n"); + errorLog.append("In node id " + node.node_id + "\n"); + fail(errorLog.toString()); + } + } + } + } + } + private void checkColumnLineage(TestCase testCase, TExecRequest execRequest, StringBuilder errorLog, StringBuilder actualOutput) { String query = testCase.getQuery();