This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 87852704516b598e463de65ddfbddca371b74140 Author: Daniel Becker <[email protected]> AuthorDate: Thu May 18 16:19:45 2023 +0200 IMPALA-12147: Allow collections of fixed length types as non-passthrough children of unions IMPALA-12019 implemented support for collections of fixed length types in the sorting tuple. This was made possible by implementing the materialisation of these collections. Building on this, this change allows such collections as non-passthrough children of UNION ALL operations. Note that plain UNIONs are not supported for any collections for other reasons and this patch does not affect them or any other set operation. Testing: Tests in nested-array-in-select-list.test and nested-map-in-select-list.test check that - the newly allowed cases work correctly and - the correct error message is given for collections of variable length types. Change-Id: I14c13323d587e5eb8a2617ecaab831c059a0fae3 Reviewed-on: http://gerrit.cloudera.org:8080/19903 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../java/org/apache/impala/planner/UnionNode.java | 11 ++-- .../QueryTest/nested-array-in-select-list.test | 62 ++++++++++++++++++-- .../QueryTest/nested-map-in-select-list.test | 66 ++++++++++++++++++++-- 3 files changed, 126 insertions(+), 13 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/planner/UnionNode.java b/fe/src/main/java/org/apache/impala/planner/UnionNode.java index 0d51fafab..a83a4ecdc 100644 --- a/fe/src/main/java/org/apache/impala/planner/UnionNode.java +++ b/fe/src/main/java/org/apache/impala/planner/UnionNode.java @@ -22,10 +22,11 @@ import java.util.List; import org.apache.impala.analysis.Analyzer; import org.apache.impala.analysis.Expr; -import org.apache.impala.analysis.TupleDescriptor; -import org.apache.impala.analysis.TupleId; import org.apache.impala.analysis.SlotDescriptor; import org.apache.impala.analysis.SlotRef; +import org.apache.impala.analysis.SortInfo; +import org.apache.impala.analysis.TupleDescriptor; +import org.apache.impala.analysis.TupleId; import org.apache.impala.thrift.TExecNodePhase; import org.apache.impala.thrift.TExplainLevel; import org.apache.impala.thrift.TExpr; @@ -262,8 +263,10 @@ public class UnionNode extends PlanNode { for (int i = 0; i < children_.size(); i++) { if (!isChildPassthrough(analyzer, children_.get(i), resultExprLists_.get(i))) { for (Expr expr : resultExprLists_.get(i)) { - Preconditions.checkState(!expr.getType().isCollectionType(), - "only pass-through UNION ALL is supported for array columns"); + Preconditions.checkState(!SortInfo.checkTypeForVarLenCollection( + expr.getType()).isPresent(), + "only pass-through UNION ALL is supported for collections of " + + "variable length types."); } newResultExprLists.add(resultExprLists_.get(i)); newChildren.add(children_.get(i)); diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test index c8db66cb9..1793c62b2 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test +++ b/testdata/workloads/functional-query/queries/QueryTest/nested-array-in-select-list.test @@ -97,19 +97,71 @@ IllegalStateException: UNION, EXCEPT and INTERSECT are not supported for collect ==== ---- QUERY # Changing a column to a different type leads "non-pass-through" union that does a -# deepcopy on the tuple, which is not yet implemented in BE for arrays. This case is -# currently caught in the planner. +# deepcopy on the tuple, which is only implemented for collections of fixed-length types +# in BE for arrays. select id, int_array from complextypestbl union all select cast(id as tinyint), int_array from complextypestbl +---- RESULTS +1,'[1,2,3]' +2,'[null,1,2,null,3,null]' +3,'[]' +4,'NULL' +5,'NULL' +6,'NULL' +7,'NULL' +8,'[-1]' +1,'[1,2,3]' +2,'[null,1,2,null,3,null]' +3,'[]' +4,'NULL' +5,'NULL' +6,'NULL' +7,'NULL' +8,'[-1]' +---- TYPES +bigint,string +==== +---- QUERY +# Changing a column to a different type leads "non-pass-through" union that does a +# deepcopy on the tuple, which is only implemented for collections of fixed-length types +# in BE for arrays. +select id, int_array_array from complextypestbl + union all select cast(id as tinyint), int_array_array from complextypestbl ---- CATCH -IllegalStateException: only pass-through UNION ALL is supported for array columns +IllegalStateException: only pass-through UNION ALL is supported for collections of variable length types. ==== ---- QUERY -# Constants in the select list of unions also lead to "non-pass-through" union. +# Constants in the select list of unions also lead to "non-pass-through" union but +# collections of fixed length types are allowed. select 1, int_array from complextypestbl union all select 2, int_array from complextypestbl; +---- RESULTS +1,'[-1]' +1,'[1,2,3]' +1,'[null,1,2,null,3,null]' +1,'[]' +1,'NULL' +1,'NULL' +1,'NULL' +1,'NULL' +2,'[-1]' +2,'[1,2,3]' +2,'[null,1,2,null,3,null]' +2,'[]' +2,'NULL' +2,'NULL' +2,'NULL' +2,'NULL' +---- TYPES +tinyint,string +==== +---- QUERY +# Constants in the select list of unions also lead to "non-pass-through" union and +# collections of variable length types are not allowed yet. +select 1, int_array_array from complextypestbl + union all select 2, int_array_array from complextypestbl; ---- CATCH -IllegalStateException: only pass-through UNION ALL is supported for array columns +IllegalStateException: only pass-through UNION ALL is supported for collections of variable length types. ==== ---- QUERY select 1 from (select int_array from complextypestbl) s diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test b/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test index 7e4312a27..114b87009 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test +++ b/testdata/workloads/functional-query/queries/QueryTest/nested-map-in-select-list.test @@ -97,19 +97,77 @@ IllegalStateException: UNION, EXCEPT and INTERSECT are not supported for collect ==== ---- QUERY # Changing a column to a different type leads "non-pass-through" union that does a -# deepcopy on the tuple, which is not yet implemented in BE for arrays. This case is -# currently caught in the planner. +# deepcopy on the tuple, which is only implemented for collections of fixed-length types +# in BE for arrays. +select id, map_int_int from map_non_varlen + union all select cast(id as tinyint), map_int_int from map_non_varlen +---- RESULTS +1,'{10:100,11:110,12:120}' +2,'{20:200,21:210,22:220}' +3,'{30:300,31:310,32:320}' +4,'{40:400,41:410,42:420}' +5,'{50:500,51:510,52:520}' +6,'{60:600,61:610,62:620}' +7,'{70:700,71:710,72:720}' +8,'{80:800,81:810,82:820}' +9,'{90:900,91:910,92:920}' +10,'{100:1000,101:1010,102:1020}' +1,'{10:100,11:110,12:120}' +2,'{20:200,21:210,22:220}' +3,'{30:300,31:310,32:320}' +4,'{40:400,41:410,42:420}' +5,'{50:500,51:510,52:520}' +6,'{60:600,61:610,62:620}' +7,'{70:700,71:710,72:720}' +8,'{80:800,81:810,82:820}' +9,'{90:900,91:910,92:920}' +10,'{100:1000,101:1010,102:1020}' +---- TYPES +int,string +==== +---- QUERY +# Changing a column to a different type leads "non-pass-through" union that does a +# deepcopy on the tuple, which is only implemented for collections of fixed-length types +# in BE for arrays. select id, int_map from complextypestbl union all select cast(id as tinyint), int_map from complextypestbl ---- CATCH -IllegalStateException: only pass-through UNION ALL is supported for array columns +IllegalStateException: only pass-through UNION ALL is supported for collections of variable length types. +==== +---- QUERY +# Constants in the select list of unions also lead to "non-pass-through" union. +select 1, map_int_int from map_non_varlen + union all select 2, map_int_int from map_non_varlen; +---- RESULTS +1,'{10:100,11:110,12:120}' +1,'{20:200,21:210,22:220}' +1,'{30:300,31:310,32:320}' +1,'{40:400,41:410,42:420}' +1,'{50:500,51:510,52:520}' +1,'{60:600,61:610,62:620}' +1,'{70:700,71:710,72:720}' +1,'{80:800,81:810,82:820}' +1,'{90:900,91:910,92:920}' +1,'{100:1000,101:1010,102:1020}' +2,'{10:100,11:110,12:120}' +2,'{20:200,21:210,22:220}' +2,'{30:300,31:310,32:320}' +2,'{40:400,41:410,42:420}' +2,'{50:500,51:510,52:520}' +2,'{60:600,61:610,62:620}' +2,'{70:700,71:710,72:720}' +2,'{80:800,81:810,82:820}' +2,'{90:900,91:910,92:920}' +2,'{100:1000,101:1010,102:1020}' +---- TYPES +tinyint,string ==== ---- QUERY # Constants in the select list of unions also lead to "non-pass-through" union. select 1, int_map from complextypestbl union all select 2, int_map from complextypestbl; ---- CATCH -IllegalStateException: only pass-through UNION ALL is supported for array columns +IllegalStateException: only pass-through UNION ALL is supported for collections of variable length types. ==== ---- QUERY select 1 from (select int_map from complextypestbl) s
