IMPALA-3643/IMPALA-5344: Fix FE tests on Java 8 This change fixes the frontend tests to make them run on Java 8, by replacing HashMap with LinkedHashMap and HashSet with LinkedHashSet where needed.
To test this I ran the frontend tests using both Oracle Java 7 and Oracle Java 8 and made sure they passed. I also verified that the tests pass with OpenJDK 7. Change-Id: Iad8e1dccec3a51293a109c420bd2b88b9d1e0625 Reviewed-on: http://gerrit.cloudera.org:8080/7073 Reviewed-by: Lars Volker <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/ae5a8770 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/ae5a8770 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/ae5a8770 Branch: refs/heads/master Commit: ae5a8770eef05f8dab025164e599eed96f033cfe Parents: 9b0d71f Author: Lars Volker <[email protected]> Authored: Sat Jun 3 21:26:28 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Wed Jun 28 01:24:11 2017 +0000 ---------------------------------------------------------------------- fe/src/main/cup/sql-parser.cup | 15 +- .../apache/impala/analysis/AnalysisContext.java | 7 +- .../org/apache/impala/analysis/Analyzer.java | 9 +- .../impala/analysis/ColumnLineageGraph.java | 44 +++- .../org/apache/impala/analysis/SortInfo.java | 5 +- .../org/apache/impala/analysis/ToSqlUtils.java | 8 +- .../impala/planner/RuntimeFilterGenerator.java | 17 +- .../apache/impala/planner/RuntimeFilterId.java | 5 +- .../org/apache/impala/analysis/ToSqlTest.java | 9 +- .../queries/PlannerTest/lineage.test | 98 ++++---- .../queries/PlannerTest/order.test | 130 +++++----- .../PlannerTest/runtime-filter-propagation.test | 4 +- .../queries/PlannerTest/tpch-nested.test | 4 +- .../queries/PlannerTest/union.test | 244 +++++++++---------- 14 files changed, 321 insertions(+), 278 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/fe/src/main/cup/sql-parser.cup ---------------------------------------------------------------------- diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup index 0ca3e2d..504cddf 100644 --- a/fe/src/main/cup/sql-parser.cup +++ b/fe/src/main/cup/sql-parser.cup @@ -24,6 +24,7 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java_cup.runtime.Symbol; @@ -1477,20 +1478,20 @@ tbl_properties ::= KW_TBLPROPERTIES LPAREN properties_map:map RPAREN {: RESULT = map; :} | /* empty */ - {: RESULT = new HashMap<String, String>(); :} + {: RESULT = new LinkedHashMap<String, String>(); :} ; serde_properties ::= KW_WITH KW_SERDEPROPERTIES LPAREN properties_map:map RPAREN {: RESULT = map; :} | /* empty */ - {: RESULT = new HashMap<String, String>(); :} + {: RESULT = new LinkedHashMap<String, String>(); :} ; properties_map ::= STRING_LITERAL:key EQUAL STRING_LITERAL:value {: - HashMap<String, String> properties = new HashMap<String, String>(); + LinkedHashMap<String, String> properties = new LinkedHashMap<String, String>(); properties.put(key, value); RESULT = properties; :} @@ -1533,7 +1534,7 @@ column_options_map ::= :} | column_option:col_option {: - Map<Option, Object> options = Maps.newHashMap(); + LinkedHashMap<Option, Object> options = Maps.newLinkedHashMap(); options.put(col_option.first, col_option.second); RESULT = options; :} @@ -1700,7 +1701,7 @@ view_column_def_list ::= view_column_def ::= ident_or_default:col_name opt_comment_val:comment {: - Map<Option, Object> options = Maps.newHashMap(); + Map<Option, Object> options = Maps.newLinkedHashMap(); if (comment != null) options.put(Option.COMMENT, comment); RESULT = new ColumnDef(col_name, null, options); :} @@ -1914,7 +1915,7 @@ function_def_args_map ::= function_def_arg_key:key EQUAL STRING_LITERAL:value {: HashMap<CreateFunctionStmtBase.OptArg, String> args = - new HashMap<CreateFunctionStmtBase.OptArg, String>(); + new LinkedHashMap<CreateFunctionStmtBase.OptArg, String>(); args.put(key, value); RESULT = args; :} @@ -1925,7 +1926,7 @@ function_def_args_map ::= RESULT = args; :} | - {: RESULT = new HashMap<CreateFunctionStmtBase.OptArg, String>(); :} + {: RESULT = new LinkedHashMap<CreateFunctionStmtBase.OptArg, String>(); :} ; // Any keys added here must also be added to the end of the precedence list. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java b/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java index efde6b0..53aa000 100644 --- a/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java +++ b/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java @@ -18,6 +18,7 @@ package org.apache.impala.analysis; import java.io.StringReader; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -449,8 +450,10 @@ public class AnalysisContext { analysisResult_.isCreateTableAsSelectStmt() || analysisResult_.isCreateViewStmt() || analysisResult_.isAlterViewStmt()) { // Map of table name to a list of privilege requests associated with that table. - // These include both table-level and column-level privilege requests. - Map<String, List<PrivilegeRequest>> tablePrivReqs = Maps.newHashMap(); + // These include both table-level and column-level privilege requests. We use a + // LinkedHashMap to preserve the order in which requests are inserted. + LinkedHashMap<String, List<PrivilegeRequest>> tablePrivReqs = + Maps.newLinkedHashMap(); // Privilege requests that are not column or table-level. List<PrivilegeRequest> otherPrivReqs = Lists.newArrayList(); // Group the registered privilege requests based on the table they reference. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/fe/src/main/java/org/apache/impala/analysis/Analyzer.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java index a3e16c4..e492aea 100644 --- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java +++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java @@ -178,7 +178,9 @@ public class Analyzer { public void setIsWithClause() { isWithClause_ = true; } public boolean isWithClause() { return isWithClause_; } - // state shared between all objects of an Analyzer tree + // State shared between all objects of an Analyzer tree. We use LinkedHashMap and + // LinkedHashSet where applicable to preserve the iteration order and make the class + // behave identical across different implementations of the JVM. // TODO: Many maps here contain properties about tuples, e.g., whether // a tuple is outer/semi joined, etc. Remove the maps in favor of making // them properties of the tuple descriptor itself. @@ -202,8 +204,9 @@ public class Analyzer { // True if at least one of the analyzers belongs to a subquery. public boolean containsSubquery = false; - // all registered conjuncts (map from expr id to conjunct) - public final Map<ExprId, Expr> conjuncts = Maps.newHashMap(); + // all registered conjuncts (map from expr id to conjunct). We use a LinkedHashMap to + // preserve the order in which conjuncts are added. + public final LinkedHashMap<ExprId, Expr> conjuncts = Maps.newLinkedHashMap(); // all registered conjuncts bound by a single tuple id; used in getBoundPredicates() public final ArrayList<ExprId> singleTidConjuncts = Lists.newArrayList(); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/fe/src/main/java/org/apache/impala/analysis/ColumnLineageGraph.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/ColumnLineageGraph.java b/fe/src/main/java/org/apache/impala/analysis/ColumnLineageGraph.java index dac4e8f..bc535ff 100644 --- a/fe/src/main/java/org/apache/impala/analysis/ColumnLineageGraph.java +++ b/fe/src/main/java/org/apache/impala/analysis/ColumnLineageGraph.java @@ -17,6 +17,7 @@ package org.apache.impala.analysis; +import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; @@ -43,6 +44,7 @@ import org.apache.impala.thrift.TVertex; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -117,8 +119,7 @@ final class Vertex implements Comparable<Vertex> { if (obj == null) return false; if (obj.getClass() != this.getClass()) return false; Vertex vertex = (Vertex) obj; - return this.id_.equals(vertex.id_) && - this.label_.equals(vertex.label_); + return this.id_.equals(vertex.id_); } public int compareTo(Vertex cmp) { return this.id_.compareTo(cmp.id_); } @@ -164,32 +165,47 @@ final class MultiEdge { edgeType_ = type; } + /** + * Return an ordered set of source vertices. + */ + private ImmutableSortedSet<Vertex> getOrderedSources() { + return ImmutableSortedSet.copyOf(sources_); + } + + /** + * Return an ordered set of target vertices. + */ + private ImmutableSortedSet<Vertex> getOrderedTargets() { + return ImmutableSortedSet.copyOf(targets_); + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); Joiner joiner = Joiner.on(","); builder.append("Sources: ["); - builder.append(joiner.join(sources_) + "]\n"); + builder.append(joiner.join(getOrderedSources()) + "]\n"); builder.append("Targets: ["); - builder.append(joiner.join(targets_) + "]\n"); + builder.append(joiner.join(getOrderedTargets()) + "]\n"); builder.append("Type: " + edgeType_); return builder.toString(); } /** * Encodes this MultiEdge object to a JSON object represented by a Map. + * Returns a LinkedHashMap to guarantee a consistent ordering of elements. */ - public Map toJson() { - Map obj = new LinkedHashMap(); + public LinkedHashMap toJson() { + LinkedHashMap obj = new LinkedHashMap(); // Add sources JSONArray sourceIds = new JSONArray(); - for (Vertex vertex: sources_) { + for (Vertex vertex: getOrderedSources()) { sourceIds.add(vertex.getVertexId()); } obj.put("sources", sourceIds); // Add targets JSONArray targetIds = new JSONArray(); - for (Vertex vertex: targets_) { + for (Vertex vertex: getOrderedTargets()) { targetIds.add(vertex.getVertexId()); } obj.put("targets", targetIds); @@ -202,11 +218,11 @@ final class MultiEdge { */ public TMultiEdge toThrift() { List<TVertex> sources = Lists.newArrayList(); - for (Vertex vertex: sources_) { + for (Vertex vertex: getOrderedSources()) { sources.add(vertex.toThrift()); } List<TVertex> targets = Lists.newArrayList(); - for (Vertex vertex: targets_) { + for (Vertex vertex: getOrderedTargets()) { targets.add(vertex.toThrift()); } if (edgeType_ == EdgeType.PROJECTION) { @@ -216,7 +232,7 @@ final class MultiEdge { } /** - * Constructs a MultiEdge object from a thrift object + * Constructs a MultiEdge object from a thrift object. */ public static MultiEdge fromThrift(TMultiEdge obj){ Set<Vertex> sources = Sets.newHashSet(); @@ -324,12 +340,14 @@ public class ColumnLineageGraph { */ private MultiEdge createMultiEdge(Set<String> targets, Set<String> sources, MultiEdge.EdgeType type) { + // createVertex() generates new IDs; we sort the input sets to make the output + // deterministic and independent of the ordering of the input sets. Set<Vertex> targetVertices = Sets.newHashSet(); - for (String target: targets) { + for (String target: ImmutableSortedSet.copyOf(targets)) { targetVertices.add(createVertex(target)); } Set<Vertex> sourceVertices = Sets.newHashSet(); - for (String source: sources) { + for (String source: ImmutableSortedSet.copyOf(sources)) { sourceVertices.add(createVertex(source)); } MultiEdge edge = new MultiEdge(sourceVertices, targetVertices, type); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/fe/src/main/java/org/apache/impala/analysis/SortInfo.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/SortInfo.java b/fe/src/main/java/org/apache/impala/analysis/SortInfo.java index 6c46231..88cb375 100644 --- a/fe/src/main/java/org/apache/impala/analysis/SortInfo.java +++ b/fe/src/main/java/org/apache/impala/analysis/SortInfo.java @@ -189,8 +189,9 @@ public class SortInfo { sortTupleExprs.addAll(substOrderBy.getLhs()); // Case 2: SlotRefs in the result and ordering exprs after substituting the - // materialized ordering exprs. - Set<SlotRef> sourceSlots = Sets.newHashSet(); + // materialized ordering exprs. Using a LinkedHashSet prevents the slots getting + // reordered unnecessarily. + Set<SlotRef> sourceSlots = Sets.newLinkedHashSet(); TreeNode.collect(Expr.substituteList(resultExprs, substOrderBy, analyzer, false), Predicates.instanceOf(SlotRef.class), sourceSlots); TreeNode.collect(Expr.substituteList(orderingExprs_, substOrderBy, analyzer, false), http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java b/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java index 68f9ec3..01517e1 100644 --- a/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java +++ b/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java @@ -19,6 +19,7 @@ package org.apache.impala.analysis; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -183,7 +184,9 @@ public class ToSqlUtils { for (ColumnDef col: innerStmt.getPartitionColumnDefs()) { partitionColsSql.add(col.getColName()); } - HashMap<String, String> properties = Maps.newHashMap(innerStmt.getTblProperties()); + // Use a LinkedHashMap to preserve the ordering of the table properties. + LinkedHashMap<String, String> properties = + Maps.newLinkedHashMap(innerStmt.getTblProperties()); removeHiddenTableProperties(properties); String kuduParamsSql = getKuduPartitionByParams(innerStmt); // TODO: Pass the correct compression, if applicable. @@ -205,7 +208,8 @@ public class ToSqlUtils { Preconditions.checkNotNull(table); if (table instanceof View) return getCreateViewSql((View)table); org.apache.hadoop.hive.metastore.api.Table msTable = table.getMetaStoreTable(); - HashMap<String, String> properties = Maps.newHashMap(msTable.getParameters()); + // Use a LinkedHashMap to preserve the ordering of the table properties. + LinkedHashMap<String, String> properties = Maps.newLinkedHashMap(msTable.getParameters()); if (properties.containsKey("transient_lastDdlTime")) { properties.remove("transient_lastDdlTime"); } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/fe/src/main/java/org/apache/impala/planner/RuntimeFilterGenerator.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/planner/RuntimeFilterGenerator.java b/fe/src/main/java/org/apache/impala/planner/RuntimeFilterGenerator.java index 133a14e..67c3532 100644 --- a/fe/src/main/java/org/apache/impala/planner/RuntimeFilterGenerator.java +++ b/fe/src/main/java/org/apache/impala/planner/RuntimeFilterGenerator.java @@ -432,14 +432,21 @@ public final class RuntimeFilterGenerator { } /** - * Returns a set of all the registered runtime filters. + * Returns a list of all the registered runtime filters, ordered by filter ID. */ - public Set<RuntimeFilter> getRuntimeFilters() { - Set<RuntimeFilter> result = Sets.newHashSet(); + public List<RuntimeFilter> getRuntimeFilters() { + Set<RuntimeFilter> resultSet = Sets.newHashSet(); for (List<RuntimeFilter> filters: runtimeFiltersByTid_.values()) { - result.addAll(filters); + resultSet.addAll(filters); } - return result; + List<RuntimeFilter> resultList = Lists.newArrayList(resultSet); + Collections.sort(resultList, new Comparator<RuntimeFilter>() { + public int compare(RuntimeFilter a, RuntimeFilter b) { + return a.getFilterId().compareTo(b.getFilterId()); + } + } + ); + return resultList; } /** http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/fe/src/main/java/org/apache/impala/planner/RuntimeFilterId.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/planner/RuntimeFilterId.java b/fe/src/main/java/org/apache/impala/planner/RuntimeFilterId.java index 9253207..8055533 100644 --- a/fe/src/main/java/org/apache/impala/planner/RuntimeFilterId.java +++ b/fe/src/main/java/org/apache/impala/planner/RuntimeFilterId.java @@ -20,7 +20,8 @@ package org.apache.impala.planner; import org.apache.impala.common.Id; import org.apache.impala.common.IdGenerator; -public class RuntimeFilterId extends Id<RuntimeFilterId> { +public class RuntimeFilterId extends Id<RuntimeFilterId> + implements Comparable<Id<RuntimeFilterId>> { // Construction only allowed via an IdGenerator. protected RuntimeFilterId(int id) { super(id); @@ -42,4 +43,6 @@ public class RuntimeFilterId extends Id<RuntimeFilterId> { @Override public int hashCode() { return id_; } + + public int compareTo(RuntimeFilterId cmp) { return Integer.compare(id_, cmp.id_); } } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java index 6660aba..8741e22 100644 --- a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java @@ -314,8 +314,8 @@ public class ToSqlTest extends FrontendTestBase { "CREATE TABLE default.p ( a BIGINT PRIMARY KEY, b TIMESTAMP " + "DEFAULT '1987-05-19' ) PARTITION BY HASH (a) PARTITIONS 3 " + "STORED AS KUDU TBLPROPERTIES ('kudu.master_addresses'='foo', " + - "'kudu.table_name'='impala::default.p', " + - "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler')", true); + "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler', " + + "'kudu.table_name'='impala::default.p')", true); } @Test @@ -346,8 +346,9 @@ public class ToSqlTest extends FrontendTestBase { "default", "CREATE TABLE default.p PRIMARY KEY (a, b) PARTITION BY HASH (a) PARTITIONS 3, " + "RANGE (b) (PARTITION VALUE = 1) STORED AS KUDU TBLPROPERTIES " + - "('kudu.master_addresses'='foo', 'kudu.table_name'='impala::default.p', " + - "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler') AS " + + "('kudu.master_addresses'='foo', " + + "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler', " + + "'kudu.table_name'='impala::default.p') AS " + "SELECT int_col a, bigint_col b FROM functional.alltypes", true); } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test b/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test index 3524287..7bb79a9 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test @@ -32,17 +32,17 @@ select * from ( { "id":1, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.int_col" + "vertexId":"functional.alltypes.bigint_col" }, { "id":2, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.tinyint_col" + "vertexId":"functional.alltypes.int_col" }, { "id":3, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.bigint_col" + "vertexId":"functional.alltypes.tinyint_col" } ] } @@ -166,27 +166,27 @@ order by b.bigint_col limit 10 { "id":8, "vertexType":"COLUMN", - "vertexId":"functional.alltypessmall.id" + "vertexId":"functional.alltypes.int_col" }, { "id":9, "vertexType":"COLUMN", - "vertexId":"functional.alltypessmall.float_col" + "vertexId":"functional.alltypes.year" }, { "id":10, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.int_col" + "vertexId":"functional.alltypessmall.bigint_col" }, { "id":11, "vertexType":"COLUMN", - "vertexId":"functional.alltypessmall.bigint_col" + "vertexId":"functional.alltypessmall.float_col" }, { "id":12, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.year" + "vertexId":"functional.alltypessmall.id" } ] } @@ -310,22 +310,22 @@ where a.year = 2009 and b.month = 2 { "id":4, "vertexType":"COLUMN", - "vertexId":"functional.alltypessmall.id" + "vertexId":"functional.alltypes.id" }, { "id":5, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.id" + "vertexId":"functional.alltypes.year" }, { "id":6, "vertexType":"COLUMN", - "vertexId":"functional.alltypessmall.month" + "vertexId":"functional.alltypessmall.id" }, { "id":7, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.year" + "vertexId":"functional.alltypessmall.month" } ] } @@ -1108,8 +1108,8 @@ where year=2009 and month=05 }, { "sources":[ - 16, - 15 + 15, + 16 ], "targets":[ 0, @@ -1346,7 +1346,6 @@ where year=2009 and month>10 ], "targets":[ 0, - 17, 2, 4, 6, @@ -1357,7 +1356,8 @@ where year=2009 and month>10 12, 13, 14, - 15 + 15, + 17 ], "edgeType":"PREDICATE" } @@ -1596,18 +1596,18 @@ having min(id) > 10 ], "targets":[ 0, - 17, 1, - 19, 3, - 21, 5, - 23, 7, 9, 11, 13, - 15 + 15, + 17, + 19, + 21, + 23 ], "edgeType":"PREDICATE" } @@ -1945,12 +1945,12 @@ order by a.tinyint_col, a.int_col { "id":6, "vertexType":"COLUMN", - "vertexId":"functional.alltypessmall.id" + "vertexId":"functional.alltypes.id" }, { "id":7, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.id" + "vertexId":"functional.alltypessmall.id" } ] } @@ -2592,12 +2592,12 @@ and x.int_col + x.float_col + cast(c.string_col as float) < 1000 { "id":12, "vertexType":"COLUMN", - "vertexId":"functional.alltypesagg.month" + "vertexId":"functional.alltypesagg.day" }, { "id":13, "vertexType":"COLUMN", - "vertexId":"functional.alltypesagg.day" + "vertexId":"functional.alltypesagg.month" } ] } @@ -3077,7 +3077,7 @@ and a.bigint_col > 10 { "id":4, "vertexType":"COLUMN", - "vertexId":"functional.alltypesagg.int_col" + "vertexId":"functional.alltypes.bigint_col" }, { "id":5, @@ -3087,7 +3087,7 @@ and a.bigint_col > 10 { "id":6, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.bigint_col" + "vertexId":"functional.alltypesagg.int_col" } ] } @@ -3410,17 +3410,17 @@ from { "id":2, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.int_col" + "vertexId":"functional.alltypes.bigint_col" }, { "id":3, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.tinyint_col" + "vertexId":"functional.alltypes.bool_col" }, { "id":4, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.bigint_col" + "vertexId":"functional.alltypes.int_col" }, { "id":5, @@ -3430,7 +3430,7 @@ from { "id":6, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.bool_col" + "vertexId":"functional.alltypes.tinyint_col" } ] } @@ -3724,12 +3724,12 @@ create view test_view_lineage (a1, a2, a3, a4, a5, a6, a7) as { "id":13, "vertexType":"COLUMN", - "vertexId":"functional.alltypesagg.month" + "vertexId":"functional.alltypesagg.day" }, { "id":14, "vertexType":"COLUMN", - "vertexId":"functional.alltypesagg.day" + "vertexId":"functional.alltypesagg.month" } ] } @@ -3853,27 +3853,27 @@ create view test_view_lineage as { "id":8, "vertexType":"COLUMN", - "vertexId":"functional.alltypessmall.id" + "vertexId":"functional.alltypes.int_col" }, { "id":9, "vertexType":"COLUMN", - "vertexId":"functional.alltypessmall.float_col" + "vertexId":"functional.alltypes.year" }, { "id":10, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.int_col" + "vertexId":"functional.alltypessmall.bigint_col" }, { "id":11, "vertexType":"COLUMN", - "vertexId":"functional.alltypessmall.bigint_col" + "vertexId":"functional.alltypessmall.float_col" }, { "id":12, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.year" + "vertexId":"functional.alltypessmall.id" } ] } @@ -3963,12 +3963,12 @@ select * from ( { "id":2, "vertexType":"COLUMN", - "vertexId":"functional.allcomplextypes.int_struct_col.f2" + "vertexId":"functional.allcomplextypes.int_struct_col.f1" }, { "id":3, "vertexType":"COLUMN", - "vertexId":"functional.allcomplextypes.int_struct_col.f1" + "vertexId":"functional.allcomplextypes.int_struct_col.f2" }, { "id":4, @@ -4255,12 +4255,12 @@ select * from functional.allcomplextypes t, t.int_array_col a, t.struct_map_col { "id":14, "vertexType":"COLUMN", - "vertexId":"functional.allcomplextypes.struct_map_col" + "vertexId":"functional.allcomplextypes.int_array_col" }, { "id":15, "vertexType":"COLUMN", - "vertexId":"functional.allcomplextypes.int_array_col" + "vertexId":"functional.allcomplextypes.struct_map_col" } ] } @@ -4313,7 +4313,7 @@ select a + b as ab, c, d, e from functional.allcomplextypes t, }, { "sources":[ - 2, + 1, 7 ], "targets":[ @@ -4332,7 +4332,7 @@ select a + b as ab, c, d, e from functional.allcomplextypes t, }, { "sources":[ - 2, + 1, 10, 11 ], @@ -4354,12 +4354,12 @@ select a + b as ab, c, d, e from functional.allcomplextypes t, { "id":1, "vertexType":"COLUMN", - "vertexId":"functional.allcomplextypes.struct_map_col.value.f1" + "vertexId":"functional.allcomplextypes.int_array_col.item" }, { "id":2, "vertexType":"COLUMN", - "vertexId":"functional.allcomplextypes.int_array_col.item" + "vertexId":"functional.allcomplextypes.struct_map_col.value.f1" }, { "id":3, @@ -4492,17 +4492,17 @@ where k.int_col < 10 { "id":1, "vertexType":"COLUMN", - "vertexId":"functional_kudu.alltypes.int_col" + "vertexId":"functional.alltypes.id" }, { "id":2, "vertexType":"COLUMN", - "vertexId":"functional.alltypes.id" + "vertexId":"functional_kudu.alltypes.id" }, { "id":3, "vertexType":"COLUMN", - "vertexId":"functional_kudu.alltypes.id" + "vertexId":"functional_kudu.alltypes.int_col" } ] } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/testdata/workloads/functional-planner/queries/PlannerTest/order.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/order.test b/testdata/workloads/functional-planner/queries/PlannerTest/order.test index 8bbcc7c..6c39366 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/order.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/order.test @@ -691,25 +691,13 @@ PLAN-ROOT SINK | order by: year ASC, month ASC, id ASC | 09:UNION -| pass-through-operands: 10,11,08 -| -|--15:TOP-N [LIMIT=3] -| | order by: id ASC -| | -| 12:UNION -| | pass-through-operands: all -| | -| |--14:SCAN HDFS [functional.alltypestiny] -| | partitions=0/4 files=0 size=0B -| | -| 13:SCAN HDFS [functional.alltypestiny] -| partitions=1/4 files=1 size=115B +| pass-through-operands: all | |--08:AGGREGATE [FINALIZE] | | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | | | 00:UNION -| | pass-through-operands: 01,02,03 +| | pass-through-operands: all | | | |--07:TOP-N [LIMIT=3] | | | order by: id ASC @@ -732,6 +720,18 @@ PLAN-ROOT SINK | 01:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | +|--15:TOP-N [LIMIT=3] +| | order by: id ASC +| | +| 12:UNION +| | pass-through-operands: all +| | +| |--14:SCAN HDFS [functional.alltypestiny] +| | partitions=0/4 files=0 size=0B +| | +| 13:SCAN HDFS [functional.alltypestiny] +| partitions=1/4 files=1 size=115B +| |--11:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | @@ -747,40 +747,22 @@ PLAN-ROOT SINK | order by: year ASC, month ASC, id ASC | 09:UNION -| pass-through-operands: 10,11,20 -| -|--22:EXCHANGE [RANDOM] -| | -| 21:MERGING-EXCHANGE [UNPARTITIONED] -| | order by: id ASC -| | limit: 3 -| | -| 15:TOP-N [LIMIT=3] -| | order by: id ASC -| | -| 12:UNION -| | pass-through-operands: all -| | -| |--14:SCAN HDFS [functional.alltypestiny] -| | partitions=0/4 files=0 size=0B -| | -| 13:SCAN HDFS [functional.alltypestiny] -| partitions=1/4 files=1 size=115B +| pass-through-operands: all | -|--20:AGGREGATE [FINALIZE] +|--21:AGGREGATE [FINALIZE] | | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | | -| 19:EXCHANGE [HASH(id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col,date_string_col,string_col,timestamp_col,year,month)] +| 20:EXCHANGE [HASH(id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col,date_string_col,string_col,timestamp_col,year,month)] | | | 08:AGGREGATE [STREAMING] | | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | | | 00:UNION -| | pass-through-operands: 01,02,03 +| | pass-through-operands: all | | -| |--18:EXCHANGE [RANDOM] +| |--19:EXCHANGE [RANDOM] | | | -| | 17:MERGING-EXCHANGE [UNPARTITIONED] +| | 18:MERGING-EXCHANGE [UNPARTITIONED] | | | order by: id ASC | | | limit: 3 | | | @@ -805,6 +787,24 @@ PLAN-ROOT SINK | 01:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | +|--22:EXCHANGE [RANDOM] +| | +| 17:MERGING-EXCHANGE [UNPARTITIONED] +| | order by: id ASC +| | limit: 3 +| | +| 15:TOP-N [LIMIT=3] +| | order by: id ASC +| | +| 12:UNION +| | pass-through-operands: all +| | +| |--14:SCAN HDFS [functional.alltypestiny] +| | partitions=0/4 files=0 size=0B +| | +| 13:SCAN HDFS [functional.alltypestiny] +| partitions=1/4 files=1 size=115B +| |--11:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | @@ -1144,20 +1144,21 @@ PLAN-ROOT SINK | order by: bigint_col ASC | 00:UNION +| pass-through-operands: 05 | -|--05:TOP-N [LIMIT=100] -| | order by: int_col ASC +|--02:TOP-N [LIMIT=100] +| | order by: tinyint_col ASC | | -| 04:AGGREGATE [FINALIZE] -| | group by: float_col, bigint_col, int_col, tinyint_col, smallint_col, id -| | -| 03:SCAN HDFS [functional.alltypes] +| 01:SCAN HDFS [functional.alltypes] | partitions=24/24 files=24 size=478.45KB | -02:TOP-N [LIMIT=100] -| order by: tinyint_col ASC +05:TOP-N [LIMIT=100] +| order by: int_col ASC | -01:SCAN HDFS [functional.alltypes] +04:AGGREGATE [FINALIZE] +| group by: float_col, bigint_col, int_col, tinyint_col, smallint_col, id +| +03:SCAN HDFS [functional.alltypes] partitions=24/24 files=24 size=478.45KB ---- DISTRIBUTEDPLAN PLAN-ROOT SINK @@ -1183,33 +1184,34 @@ PLAN-ROOT SINK | order by: bigint_col ASC | 00:UNION +| pass-through-operands: 14 | |--15:MERGING-EXCHANGE [UNPARTITIONED] -| | order by: int_col ASC +| | order by: tinyint_col ASC | | limit: 100 | | -| 05:TOP-N [LIMIT=100] -| | order by: int_col ASC -| | -| 14:AGGREGATE [FINALIZE] -| | group by: float_col, bigint_col, int_col, tinyint_col, smallint_col, id -| | -| 13:EXCHANGE [HASH(float_col,bigint_col,int_col,tinyint_col,smallint_col,id)] +| 02:TOP-N [LIMIT=100] +| | order by: tinyint_col ASC | | -| 04:AGGREGATE [STREAMING] -| | group by: float_col, bigint_col, int_col, tinyint_col, smallint_col, id -| | -| 03:SCAN HDFS [functional.alltypes] +| 01:SCAN HDFS [functional.alltypes] | partitions=24/24 files=24 size=478.45KB | -12:MERGING-EXCHANGE [UNPARTITIONED] -| order by: tinyint_col ASC +14:MERGING-EXCHANGE [UNPARTITIONED] +| order by: int_col ASC | limit: 100 | -02:TOP-N [LIMIT=100] -| order by: tinyint_col ASC +05:TOP-N [LIMIT=100] +| order by: int_col ASC +| +13:AGGREGATE [FINALIZE] +| group by: float_col, bigint_col, int_col, tinyint_col, smallint_col, id | -01:SCAN HDFS [functional.alltypes] +12:EXCHANGE [HASH(float_col,bigint_col,int_col,tinyint_col,smallint_col,id)] +| +04:AGGREGATE [STREAMING] +| group by: float_col, bigint_col, int_col, tinyint_col, smallint_col, id +| +03:SCAN HDFS [functional.alltypes] partitions=24/24 files=24 size=478.45KB ==== # Test slot materialization IMPALA-1006 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/testdata/workloads/functional-planner/queries/PlannerTest/runtime-filter-propagation.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/runtime-filter-propagation.test b/testdata/workloads/functional-planner/queries/PlannerTest/runtime-filter-propagation.test index 7f18c84..602505b 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/runtime-filter-propagation.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/runtime-filter-propagation.test @@ -1363,14 +1363,14 @@ PLAN-ROOT SINK | |--11:HASH JOIN [INNER JOIN] | | hash predicates: a.bigint_col = b.bigint_col, a.bool_col = b.bool_col, a.double_col = b.double_col, a.float_col = b.float_col, a.id = b.id, a.int_col = b.int_col, a.smallint_col = b.smallint_col, a.tinyint_col = b.tinyint_col -| | runtime filters: RF017 <- b.bool_col, RF016 <- b.bigint_col, RF019 <- b.float_col, RF018 <- b.double_col, RF021 <- b.int_col, RF020 <- b.id, RF023 <- b.tinyint_col, RF022 <- b.smallint_col +| | runtime filters: RF016 <- b.bigint_col, RF017 <- b.bool_col, RF018 <- b.double_col, RF019 <- b.float_col, RF020 <- b.id, RF021 <- b.int_col, RF022 <- b.smallint_col, RF023 <- b.tinyint_col | | | |--10:SCAN HDFS [functional.alltypestiny b] | | partitions=4/4 files=4 size=460B | | | 09:SCAN HDFS [functional.alltypes a] | partitions=24/24 files=24 size=478.45KB -| runtime filters: RF017 -> a.bool_col, RF016 -> a.bigint_col, RF019 -> a.float_col, RF018 -> a.double_col, RF021 -> a.int_col, RF020 -> a.id, RF023 -> a.tinyint_col, RF022 -> a.smallint_col +| runtime filters: RF016 -> a.bigint_col, RF017 -> a.bool_col, RF018 -> a.double_col, RF019 -> a.float_col, RF020 -> a.id, RF021 -> a.int_col, RF022 -> a.smallint_col, RF023 -> a.tinyint_col | 30:NESTED LOOP JOIN [CROSS JOIN] | http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/testdata/workloads/functional-planner/queries/PlannerTest/tpch-nested.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpch-nested.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-nested.test index 31e5d40..7b9ff13 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/tpch-nested.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-nested.test @@ -2408,7 +2408,7 @@ PLAN-ROOT SINK 01:SCAN HDFS [tpch_nested_parquet.customer c] partitions=1/1 files=4 size=577.87MB predicates: !empty(c.c_orders) - predicates on o: o_orderstatus = 'F', !empty(o.o_lineitems) + predicates on o: !empty(o.o_lineitems), o_orderstatus = 'F' predicates on l1: l1.l_receiptdate > l1.l_commitdate predicates on l3: l3.l_receiptdate > l3.l_commitdate ---- DISTRIBUTEDPLAN @@ -2483,7 +2483,7 @@ PLAN-ROOT SINK 01:SCAN HDFS [tpch_nested_parquet.customer c] partitions=1/1 files=4 size=577.87MB predicates: !empty(c.c_orders) - predicates on o: o_orderstatus = 'F', !empty(o.o_lineitems) + predicates on o: !empty(o.o_lineitems), o_orderstatus = 'F' predicates on l1: l1.l_receiptdate > l1.l_commitdate predicates on l3: l3.l_receiptdate > l3.l_commitdate ==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ae5a8770/testdata/workloads/functional-planner/queries/PlannerTest/union.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/union.test b/testdata/workloads/functional-planner/queries/PlannerTest/union.test index b2368be..03a2df5 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/union.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/union.test @@ -106,7 +106,11 @@ select * from functional.alltypestiny where year=2009 and month=2 limit 1 PLAN-ROOT SINK | 00:UNION -| pass-through-operands: 01,04 +| pass-through-operands: all +| +|--04:SCAN HDFS [functional.alltypestiny] +| partitions=1/4 files=1 size=115B +| limit: 1 | |--03:TOP-N [LIMIT=1] | | order by: int_col ASC @@ -114,10 +118,6 @@ PLAN-ROOT SINK | 02:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | -|--04:SCAN HDFS [functional.alltypestiny] -| partitions=1/4 files=1 size=115B -| limit: 1 -| 01:SCAN HDFS [functional.alltypestiny] partitions=1/4 files=1 size=115B limit: 1 @@ -132,9 +132,16 @@ NODE 4: PLAN-ROOT SINK | 00:UNION -| pass-through-operands: 05,06 +| pass-through-operands: all | -|--07:MERGING-EXCHANGE [UNPARTITIONED] +|--07:EXCHANGE [UNPARTITIONED] +| | limit: 1 +| | +| 04:SCAN HDFS [functional.alltypestiny] +| partitions=1/4 files=1 size=115B +| limit: 1 +| +|--06:MERGING-EXCHANGE [UNPARTITIONED] | | order by: int_col ASC | | limit: 1 | | @@ -144,13 +151,6 @@ PLAN-ROOT SINK | 02:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | -|--06:EXCHANGE [UNPARTITIONED] -| | limit: 1 -| | -| 04:SCAN HDFS [functional.alltypestiny] -| partitions=1/4 files=1 size=115B -| limit: 1 -| 05:EXCHANGE [UNPARTITIONED] | limit: 1 | @@ -1103,21 +1103,21 @@ select * from functional.alltypestiny where year=2009 and month=1 PLAN-ROOT SINK | 00:UNION -| pass-through-operands: 05 +| pass-through-operands: all | -|--04:TOP-N [LIMIT=3] -| | order by: tinyint_col ASC -| | -| 01:UNION -| | pass-through-operands: all -| | -| |--03:SCAN HDFS [functional.alltypestiny] -| | partitions=1/4 files=1 size=115B -| | -| 02:SCAN HDFS [functional.alltypestiny] +|--05:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | -05:SCAN HDFS [functional.alltypestiny] +04:TOP-N [LIMIT=3] +| order by: tinyint_col ASC +| +01:UNION +| pass-through-operands: all +| +|--03:SCAN HDFS [functional.alltypestiny] +| partitions=1/4 files=1 size=115B +| +02:SCAN HDFS [functional.alltypestiny] partitions=1/4 files=1 size=115B ---- SCANRANGELOCATIONS NODE 2: @@ -1132,27 +1132,27 @@ PLAN-ROOT SINK 08:EXCHANGE [UNPARTITIONED] | 00:UNION -| pass-through-operands: 05 +| pass-through-operands: all | -|--07:EXCHANGE [RANDOM] -| | -| 06:MERGING-EXCHANGE [UNPARTITIONED] -| | order by: tinyint_col ASC -| | limit: 3 -| | -| 04:TOP-N [LIMIT=3] -| | order by: tinyint_col ASC -| | -| 01:UNION -| | pass-through-operands: all -| | -| |--03:SCAN HDFS [functional.alltypestiny] -| | partitions=1/4 files=1 size=115B -| | -| 02:SCAN HDFS [functional.alltypestiny] +|--05:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | -05:SCAN HDFS [functional.alltypestiny] +07:EXCHANGE [RANDOM] +| +06:MERGING-EXCHANGE [UNPARTITIONED] +| order by: tinyint_col ASC +| limit: 3 +| +04:TOP-N [LIMIT=3] +| order by: tinyint_col ASC +| +01:UNION +| pass-through-operands: all +| +|--03:SCAN HDFS [functional.alltypestiny] +| partitions=1/4 files=1 size=115B +| +02:SCAN HDFS [functional.alltypestiny] partitions=1/4 files=1 size=115B ==== // Union unnesting: UNION ALL doesn't absorb nested union with order by and limit, @@ -1167,7 +1167,7 @@ union all PLAN-ROOT SINK | 00:UNION -| pass-through-operands: 01 +| pass-through-operands: all | |--05:TOP-N [LIMIT=3] | | order by: tinyint_col ASC @@ -1196,7 +1196,7 @@ PLAN-ROOT SINK 08:EXCHANGE [UNPARTITIONED] | 00:UNION -| pass-through-operands: 01 +| pass-through-operands: all | |--07:EXCHANGE [RANDOM] | | @@ -1478,21 +1478,21 @@ PLAN-ROOT SINK | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | 00:UNION -| pass-through-operands: 05 +| pass-through-operands: all | -|--04:TOP-N [LIMIT=3] -| | order by: tinyint_col ASC -| | -| 01:UNION -| | pass-through-operands: all -| | -| |--03:SCAN HDFS [functional.alltypestiny] -| | partitions=1/4 files=1 size=115B -| | -| 02:SCAN HDFS [functional.alltypestiny] +|--05:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | -05:SCAN HDFS [functional.alltypestiny] +04:TOP-N [LIMIT=3] +| order by: tinyint_col ASC +| +01:UNION +| pass-through-operands: all +| +|--03:SCAN HDFS [functional.alltypestiny] +| partitions=1/4 files=1 size=115B +| +02:SCAN HDFS [functional.alltypestiny] partitions=1/4 files=1 size=115B ---- SCANRANGELOCATIONS NODE 2: @@ -1515,27 +1515,27 @@ PLAN-ROOT SINK | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | 00:UNION -| pass-through-operands: 05 +| pass-through-operands: all | -|--08:EXCHANGE [RANDOM] -| | -| 07:MERGING-EXCHANGE [UNPARTITIONED] -| | order by: tinyint_col ASC -| | limit: 3 -| | -| 04:TOP-N [LIMIT=3] -| | order by: tinyint_col ASC -| | -| 01:UNION -| | pass-through-operands: all -| | -| |--03:SCAN HDFS [functional.alltypestiny] -| | partitions=1/4 files=1 size=115B -| | -| 02:SCAN HDFS [functional.alltypestiny] +|--05:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | -05:SCAN HDFS [functional.alltypestiny] +08:EXCHANGE [RANDOM] +| +07:MERGING-EXCHANGE [UNPARTITIONED] +| order by: tinyint_col ASC +| limit: 3 +| +04:TOP-N [LIMIT=3] +| order by: tinyint_col ASC +| +01:UNION +| pass-through-operands: all +| +|--03:SCAN HDFS [functional.alltypestiny] +| partitions=1/4 files=1 size=115B +| +02:SCAN HDFS [functional.alltypestiny] partitions=1/4 files=1 size=115B ==== // Union unnesting: UNION DISTINCT doesn't absorb nested union with order by and limit @@ -1553,7 +1553,7 @@ PLAN-ROOT SINK | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | 00:UNION -| pass-through-operands: 01 +| pass-through-operands: all | |--05:TOP-N [LIMIT=3] | | order by: tinyint_col ASC @@ -1590,7 +1590,7 @@ PLAN-ROOT SINK | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | 00:UNION -| pass-through-operands: 01 +| pass-through-operands: all | |--08:EXCHANGE [RANDOM] | | @@ -1773,7 +1773,7 @@ PLAN-ROOT SINK | | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | | | 02:UNION -| | pass-through-operands: 03,04 +| | pass-through-operands: all | | | |--09:TOP-N [LIMIT=3] | | | order by: tinyint_col ASC @@ -1826,7 +1826,7 @@ PLAN-ROOT SINK | | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | | | 02:UNION -| | pass-through-operands: 03,04 +| | pass-through-operands: all | | | |--14:EXCHANGE [RANDOM] | | | @@ -1881,7 +1881,7 @@ PLAN-ROOT SINK | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | 00:UNION -| pass-through-operands: 01,02,03 +| pass-through-operands: all | |--08:TOP-N [LIMIT=3] | | order by: tinyint_col ASC @@ -1931,7 +1931,7 @@ PLAN-ROOT SINK | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | 00:UNION -| pass-through-operands: 01,02,03 +| pass-through-operands: all | |--13:EXCHANGE [RANDOM] | | @@ -1992,25 +1992,13 @@ union all PLAN-ROOT SINK | 09:UNION -| pass-through-operands: 10,11,08 -| -|--15:TOP-N [LIMIT=3] -| | order by: tinyint_col ASC -| | -| 12:UNION -| | pass-through-operands: all -| | -| |--14:SCAN HDFS [functional.alltypestiny] -| | partitions=0/4 files=0 size=0B -| | -| 13:SCAN HDFS [functional.alltypestiny] -| partitions=1/4 files=1 size=115B +| pass-through-operands: all | |--08:AGGREGATE [FINALIZE] | | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | | | 00:UNION -| | pass-through-operands: 01,02,03 +| | pass-through-operands: all | | | |--07:TOP-N [LIMIT=3] | | | order by: tinyint_col ASC @@ -2033,6 +2021,18 @@ PLAN-ROOT SINK | 01:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | +|--15:TOP-N [LIMIT=3] +| | order by: tinyint_col ASC +| | +| 12:UNION +| | pass-through-operands: all +| | +| |--14:SCAN HDFS [functional.alltypestiny] +| | partitions=0/4 files=0 size=0B +| | +| 13:SCAN HDFS [functional.alltypestiny] +| partitions=1/4 files=1 size=115B +| |--11:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | @@ -2062,40 +2062,22 @@ PLAN-ROOT SINK 22:EXCHANGE [UNPARTITIONED] | 09:UNION -| pass-through-operands: 10,11,19 -| -|--21:EXCHANGE [RANDOM] -| | -| 20:MERGING-EXCHANGE [UNPARTITIONED] -| | order by: tinyint_col ASC -| | limit: 3 -| | -| 15:TOP-N [LIMIT=3] -| | order by: tinyint_col ASC -| | -| 12:UNION -| | pass-through-operands: all -| | -| |--14:SCAN HDFS [functional.alltypestiny] -| | partitions=0/4 files=0 size=0B -| | -| 13:SCAN HDFS [functional.alltypestiny] -| partitions=1/4 files=1 size=115B +| pass-through-operands: all | -|--19:AGGREGATE [FINALIZE] +|--20:AGGREGATE [FINALIZE] | | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | | -| 18:EXCHANGE [HASH(id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col,date_string_col,string_col,timestamp_col,year,month)] +| 19:EXCHANGE [HASH(id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col,date_string_col,string_col,timestamp_col,year,month)] | | | 08:AGGREGATE [STREAMING] | | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | | | 00:UNION -| | pass-through-operands: 01,02,03 +| | pass-through-operands: all | | -| |--17:EXCHANGE [RANDOM] +| |--18:EXCHANGE [RANDOM] | | | -| | 16:MERGING-EXCHANGE [UNPARTITIONED] +| | 17:MERGING-EXCHANGE [UNPARTITIONED] | | | order by: tinyint_col ASC | | | limit: 3 | | | @@ -2120,6 +2102,24 @@ PLAN-ROOT SINK | 01:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | +|--21:EXCHANGE [RANDOM] +| | +| 16:MERGING-EXCHANGE [UNPARTITIONED] +| | order by: tinyint_col ASC +| | limit: 3 +| | +| 15:TOP-N [LIMIT=3] +| | order by: tinyint_col ASC +| | +| 12:UNION +| | pass-through-operands: all +| | +| |--14:SCAN HDFS [functional.alltypestiny] +| | partitions=0/4 files=0 size=0B +| | +| 13:SCAN HDFS [functional.alltypestiny] +| partitions=1/4 files=1 size=115B +| |--11:SCAN HDFS [functional.alltypestiny] | partitions=1/4 files=1 size=115B | @@ -3028,7 +3028,7 @@ where v.id < 10 and v.int_col > 20 PLAN-ROOT SINK | 00:UNION -| pass-through-operands: 01,02 +| pass-through-operands: all | |--08:SELECT | | predicates: id < 10, int_col > 20 @@ -3073,7 +3073,7 @@ PLAN-ROOT SINK | group by: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month | 00:UNION -| pass-through-operands: 01,02 +| pass-through-operands: all | |--08:SELECT | | predicates: id < 10, int_col > 20
