Repository: tajo Updated Branches: refs/heads/master 5c852b798 -> af4083bfd
TAJO-1232: Implicit groupby queries with LIMIT lead to wrong results. (jihoon) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/af4083bf Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/af4083bf Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/af4083bf Branch: refs/heads/master Commit: af4083bfd337bfd6fe8278b0363c987d19088e50 Parents: 5c852b7 Author: Jihoon Son <[email protected]> Authored: Thu Dec 18 16:50:09 2014 +0900 Committer: Jihoon Son <[email protected]> Committed: Thu Dec 18 16:50:09 2014 +0900 ---------------------------------------------------------------------- CHANGES | 3 +++ .../org/apache/tajo/master/GlobalEngine.java | 8 ++++---- .../tajo/engine/query/TestGroupByQuery.java | 21 ++++++++++++++++++++ .../TestGroupByQuery/testGroupbyWithLimit1.sql | 1 + .../TestGroupByQuery/testGroupbyWithLimit2.sql | 1 + .../TestGroupByQuery/testGroupbyWithLimit3.sql | 1 + .../testGroupbyWithLimit1.result | 3 +++ .../testGroupbyWithLimit2.result | 3 +++ .../testGroupbyWithLimit3.result | 3 +++ .../org/apache/tajo/plan/LogicalPlanner.java | 7 +++++++ 10 files changed, 47 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 2230d0f..29b66d1 100644 --- a/CHANGES +++ b/CHANGES @@ -112,6 +112,9 @@ Release 0.9.1 - unreleased BUG FIXES + TAJO-1232: Implicit groupby queries with LIMIT lead to wrong results. + (jihoon) + TAJO-1254: Fix getProgress race conditions in Query. (jinho) TAJO-1252: PathValidator should allow hdfs paths which contain IP addresses. http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index a9624f8..71b1f9b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -204,10 +204,10 @@ public class GlobalEngine extends AbstractService { } private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, - Session session, - LogicalPlan plan, - String sql, - String jsonExpr) throws Exception { + Session session, + LogicalPlan plan, + String sql, + String jsonExpr) throws Exception { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java index 3dd1219..edbe029 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java @@ -749,4 +749,25 @@ public class TestGroupByQuery extends QueryTestCaseBase { ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME.defaultVal); } } + + @Test + public final void testGroupbyWithLimit1() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testGroupbyWithLimit2() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testGroupbyWithLimit3() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql new file mode 100644 index 0000000..06d96af --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql @@ -0,0 +1 @@ +select sum(l_linenumber) from lineitem limit 1; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql new file mode 100644 index 0000000..1ddb171 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql @@ -0,0 +1 @@ +select sum(l_linenumber) from lineitem limit 10; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql new file mode 100644 index 0000000..1bb7c7f --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql @@ -0,0 +1 @@ +select l_orderkey, sum(l_linenumber) from lineitem group by l_orderkey limit 1; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result new file mode 100644 index 0000000..23bbaea --- /dev/null +++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result @@ -0,0 +1,3 @@ +?sum +------------------------------- +7 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result new file mode 100644 index 0000000..23bbaea --- /dev/null +++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result @@ -0,0 +1,3 @@ +?sum +------------------------------- +7 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result new file mode 100644 index 0000000..cb1e141 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result @@ -0,0 +1,3 @@ +l_orderkey,?sum +------------------------------- +3,3 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index df45c3a..1a426e0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -716,6 +716,13 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex LogicalPlan plan = context.plan; QueryBlock block = context.queryBlock; + + // The limit operation must affect to the number of results, not the number of input records. + // Thus, the aggregation must be carried out before the limit operation. + if (child.getType() == NodeType.LIMIT) { + child = ((LimitNode)child).getChild(); + } + GroupbyNode groupbyNode = context.plan.createNode(GroupbyNode.class); groupbyNode.setChild(child); groupbyNode.setInSchema(child.getOutSchema());
