This is an automated email from the ASF dual-hosted git repository.
luoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 2e96919 DRILL-7893: Column alias is not working for a parquet file
2e96919 is described below
commit 2e96919142e0ff44b64a386d38b1244e80498e8f
Author: Volodymyr Vysotskyi <[email protected]>
AuthorDate: Wed Apr 28 21:50:08 2021 +0300
DRILL-7893: Column alias is not working for a parquet file
---
.../physical/visitor/ComplexToJsonPrelVisitor.java | 3 +--
.../physical/visitor/ExcessiveExchangeIdentifier.java | 15 +++++++++++++--
.../planner/physical/visitor/FinalColumnReorderer.java | 17 +++++++++++------
.../physical/visitor/InsertLocalExchangeVisitor.java | 3 +++
.../physical/visitor/LateralUnnestRowIDVisitor.java | 4 +++-
.../physical/visitor/RewriteProjectToFlatten.java | 13 +++++++++----
.../planner/physical/visitor/RuntimeFilterVisitor.java | 11 +++++++----
.../physical/visitor/SelectionVectorPrelVisitor.java | 5 ++++-
.../physical/visitor/SplitUpComplexExpressions.java | 3 +++
.../exec/planner/sql/handlers/DefaultSqlHandler.java | 10 ++++++++--
.../exec/store/parquet/TestParquetFilterPushDown.java | 13 +++++++++++++
11 files changed, 75 insertions(+), 22 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ComplexToJsonPrelVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ComplexToJsonPrelVisitor.java
index eb59611..8186351 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ComplexToJsonPrelVisitor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ComplexToJsonPrelVisitor.java
@@ -22,7 +22,6 @@ import java.util.Collections;
import org.apache.drill.exec.planner.physical.ComplexToJsonPrel;
import org.apache.drill.exec.planner.physical.Prel;
import org.apache.drill.exec.planner.physical.ScreenPrel;
-import org.apache.calcite.rel.RelNode;
public class ComplexToJsonPrelVisitor extends BasePrelVisitor<Prel, Void,
RuntimeException> {
@@ -34,7 +33,7 @@ public class ComplexToJsonPrelVisitor extends
BasePrelVisitor<Prel, Void, Runtim
@Override
public Prel visitScreen(ScreenPrel prel, Void value) throws RuntimeException
{
- return prel.copy(prel.getTraitSet(),
Collections.singletonList((RelNode)new
ComplexToJsonPrel((Prel)prel.getInput())));
+ return prel.copy(prel.getTraitSet(), Collections.singletonList(new
ComplexToJsonPrel((Prel) prel.getInput())));
}
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
index 3660868..18a34b7 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
@@ -43,7 +43,7 @@ public class ExcessiveExchangeIdentifier extends
BasePrelVisitor<Prel, Excessive
this.targetSliceSize = targetSliceSize;
}
- public static Prel removeExcessiveEchanges(Prel prel, long targetSliceSize) {
+ public static Prel removeExcessiveExchanges(Prel prel, long targetSliceSize)
{
ExcessiveExchangeIdentifier exchange = new
ExcessiveExchangeIdentifier(targetSliceSize);
return prel.accept(exchange, exchange.getNewStat());
}
@@ -67,8 +67,10 @@ public class ExcessiveExchangeIdentifier extends
BasePrelVisitor<Prel, Excessive
if (canRemoveExchange(parent, newFrag)) {
return newChild;
+ } else if (newChild != prel.getInput()) {
+ return (Prel) prel.copy(prel.getTraitSet(),
Collections.singletonList(newChild));
} else {
- return (Prel) prel.copy(prel.getTraitSet(),
Collections.singletonList((RelNode) newChild));
+ return prel;
}
}
@@ -85,6 +87,9 @@ public class ExcessiveExchangeIdentifier extends
BasePrelVisitor<Prel, Excessive
public Prel visitScreen(ScreenPrel prel, MajorFragmentStat s) throws
RuntimeException {
s.addScreen(prel);
RelNode child = ((Prel) prel.getInput()).accept(this, s);
+ if (child == prel.getInput()) {
+ return prel;
+ }
return prel.copy(prel.getTraitSet(), Collections.singletonList(child));
}
@@ -119,6 +124,9 @@ public class ExcessiveExchangeIdentifier extends
BasePrelVisitor<Prel, Excessive
topMostLateralJoin = null;
s.setRightSideOfLateral(false);
}
+ if (children.equals(prel.getInputs())) {
+ return prel;
+ }
return (Prel) prel.copy(prel.getTraitSet(), children);
}
@@ -146,6 +154,9 @@ public class ExcessiveExchangeIdentifier extends
BasePrelVisitor<Prel, Excessive
for (Prel p : prel) {
children.add(p.accept(this, s));
}
+ if (children.equals(prel.getInputs())) {
+ return prel;
+ }
return (Prel) prel.copy(prel.getTraitSet(), children);
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java
index b04ae43..144b62c 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java
@@ -32,10 +32,9 @@ import org.apache.calcite.rex.RexNode;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-public class FinalColumnReorderer extends BasePrelVisitor<Prel, Void,
RuntimeException>{
- static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(FinalColumnReorderer.class);
+public class FinalColumnReorderer extends BasePrelVisitor<Prel, Void,
RuntimeException> {
- private static FinalColumnReorderer INSTANCE = new FinalColumnReorderer();
+ private static final FinalColumnReorderer INSTANCE = new
FinalColumnReorderer();
public static Prel addFinalColumnOrdering(Prel prel) {
return prel.accept(INSTANCE, null);
@@ -43,8 +42,11 @@ public class FinalColumnReorderer extends
BasePrelVisitor<Prel, Void, RuntimeExc
@Override
public Prel visitScreen(ScreenPrel prel, Void value) throws RuntimeException
{
- Prel newChild = ((Prel) prel.getInput()).accept(this, value);
- return prel.copy(prel.getTraitSet(), Collections.singletonList( (RelNode)
addTrivialOrderedProjectPrel(newChild, true)));
+ Prel newChild = addTrivialOrderedProjectPrel(((Prel)
prel.getInput()).accept(this, value), true);
+ if (newChild == prel.getInput()) {
+ return prel;
+ }
+ return prel.copy(prel.getTraitSet(), Collections.singletonList(newChild));
}
private Prel addTrivialOrderedProjectPrel(Prel prel) {
@@ -76,7 +78,10 @@ public class FinalColumnReorderer extends
BasePrelVisitor<Prel, Void, RuntimeExc
@Override
public Prel visitWriter(WriterPrel prel, Void value) throws RuntimeException
{
Prel newChild = ((Prel) prel.getInput()).accept(this, null);
- return prel.copy(prel.getTraitSet(), Collections.singletonList( (RelNode)
addTrivialOrderedProjectPrel(newChild, true)));
+ if (newChild == prel.getInput()) {
+ return prel;
+ }
+ return prel.copy(prel.getTraitSet(),
Collections.singletonList(addTrivialOrderedProjectPrel(newChild, true)));
}
@Override
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
index fd15cc7..899417c 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
@@ -60,6 +60,9 @@ public class InsertLocalExchangeVisitor extends
BasePrelVisitor<Prel, Void, Runt
for(Prel child : prel){
children.add(child.accept(this, null));
}
+ if (children.equals(prel.getInputs())) {
+ return prel;
+ }
return (Prel) prel.copy(prel.getTraitSet(), children);
}
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
index 2eb4405..3cb4093 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
@@ -40,7 +40,7 @@ import java.util.Map;
*/
public class LateralUnnestRowIDVisitor extends BasePrelVisitor<Prel, Boolean,
RuntimeException> {
- private static LateralUnnestRowIDVisitor INSTANCE = new
LateralUnnestRowIDVisitor();
+ private static final LateralUnnestRowIDVisitor INSTANCE = new
LateralUnnestRowIDVisitor();
public static Prel insertRowID(Prel prel){
return prel.accept(INSTANCE, false);
@@ -51,6 +51,8 @@ public class LateralUnnestRowIDVisitor extends
BasePrelVisitor<Prel, Boolean, Ru
List<RelNode> children = getChildren(prel, isRightOfLateral);
if (isRightOfLateral) {
return prel.prepareForLateralUnnestPipeline(children);
+ } else if (children.equals(prel.getInputs())) {
+ return prel;
} else {
return (Prel) prel.copy(prel.getTraitSet(), children);
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
index f21929b..a1ffe2e 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
@@ -55,13 +55,15 @@ public class RewriteProjectToFlatten extends
BasePrelVisitor<Prel, Object, RelCo
child = child.accept(this, null);
children.add(child);
}
+ if (children.equals(prel.getInputs())) {
+ return prel;
+ }
return (Prel) prel.copy(prel.getTraitSet(), children);
}
@Override
- public Prel visitProject(ProjectPrel node, Object unused) throws
RelConversionException {
- ProjectPrel project = node;
+ public Prel visitProject(ProjectPrel project, Object unused) throws
RelConversionException {
List<RexNode> exprList = new ArrayList<>();
boolean rewrite = false;
@@ -88,15 +90,18 @@ public class RewriteProjectToFlatten extends
BasePrelVisitor<Prel, Object, RelCo
i++;
exprList.add(newExpr);
}
- if (rewrite == true) {
+ if (rewrite) {
// TODO - figure out what is the right setting for the traits
Prel newChild = ((Prel)project.getInput(0)).accept(this, null);
- ProjectPrel newProject = new ProjectPrel(node.getCluster(),
project.getTraitSet(), newChild, exprList, new RelRecordType(relDataTypes));
+ ProjectPrel newProject = new ProjectPrel(project.getCluster(),
project.getTraitSet(), newChild, exprList, new RelRecordType(relDataTypes));
FlattenPrel flatten = new FlattenPrel(project.getCluster(),
project.getTraitSet(), newProject, flatttenExpr);
return flatten;
}
Prel child = ((Prel)project.getInput()).accept(this, null);
+ if (child == project.getInput() &&
exprList.equals(project.getChildExps())) {
+ return project;
+ }
return (Prel) project.copy(project.getTraitSet(), child, exprList, new
RelRecordType(relDataTypes));
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RuntimeFilterVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RuntimeFilterVisitor.java
index acc5977..427044d 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RuntimeFilterVisitor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RuntimeFilterVisitor.java
@@ -57,13 +57,13 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class RuntimeFilterVisitor extends BasePrelVisitor<Prel, Void,
RuntimeException> {
- private Set<ScanPrel> toAddRuntimeFilter = new HashSet<>();
+ private final Set<ScanPrel> toAddRuntimeFilter = new HashSet<>();
- private Multimap<ScanPrel, HashJoinPrel> probeSideScan2hj =
HashMultimap.create();
+ private final Multimap<ScanPrel, HashJoinPrel> probeSideScan2hj =
HashMultimap.create();
- private double fpp;
+ private final double fpp;
- private int bloomFilterMaxSizeInBytesDef;
+ private final int bloomFilterMaxSizeInBytesDef;
private static final AtomicLong rfIdCounter = new AtomicLong();
@@ -87,6 +87,9 @@ public class RuntimeFilterVisitor extends
BasePrelVisitor<Prel, Void, RuntimeExc
child = child.accept(this, value);
children.add(child);
}
+ if (children.equals(prel.getInputs())) {
+ return prel;
+ }
return (Prel) prel.copy(prel.getTraitSet(), children);
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SelectionVectorPrelVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SelectionVectorPrelVisitor.java
index 62b38d5..d56d3d7 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SelectionVectorPrelVisitor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SelectionVectorPrelVisitor.java
@@ -29,7 +29,7 @@ import
org.apache.drill.shaded.guava.com.google.common.collect.Lists;
public class SelectionVectorPrelVisitor extends BasePrelVisitor<Prel, Void,
RuntimeException>{
- private static SelectionVectorPrelVisitor INSTANCE = new
SelectionVectorPrelVisitor();
+ private static final SelectionVectorPrelVisitor INSTANCE = new
SelectionVectorPrelVisitor();
public static Prel addSelectionRemoversWhereNecessary(Prel prel) {
return prel.accept(INSTANCE, null);
@@ -44,6 +44,9 @@ public class SelectionVectorPrelVisitor extends
BasePrelVisitor<Prel, Void, Runt
children.add(convert(encodings, child));
}
+ if (children.equals(prel.getInputs())) {
+ return prel;
+ }
return (Prel) prel.copy(prel.getTraitSet(), children);
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
index 1f46e46..7bbd55b 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
@@ -62,6 +62,9 @@ public class SplitUpComplexExpressions extends
BasePrelVisitor<Prel, Object, Rel
child = child.accept(this, unused);
children.add(child);
}
+ if (children.equals(prel.getInputs())) {
+ return prel;
+ }
return (Prel) prel.copy(prel.getTraitSet(), children);
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index 2a8e272..baa77b8 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -544,7 +544,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
* If two fragments are both estimated to be parallelization one, remove
the exchange
* separating them.
*/
- phyRelNode =
ExcessiveExchangeIdentifier.removeExcessiveEchanges(phyRelNode,
targetSliceSize);
+ phyRelNode =
ExcessiveExchangeIdentifier.removeExcessiveExchanges(phyRelNode,
targetSliceSize);
/* Insert the IMPLICIT_COLUMN in the lateral unnest pipeline */
phyRelNode = LateralUnnestRowIDVisitor.insertRowID(phyRelNode);
@@ -589,7 +589,13 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
*/
phyRelNode =
SelectionVectorPrelVisitor.addSelectionRemoversWhereNecessary(phyRelNode);
- /* 10.)
+ /*
+ * 10.)
+ * Insert project above the screen operator or writer to ensure that final
output column names are preserved after all optimizations.
+ */
+ phyRelNode = TopProjectVisitor.insertTopProject(phyRelNode,
validatedRowType);
+
+ /* 11.)
* Finally, Make sure that the no rels are repeats.
* This could happen in the case of querying the same table twice as Optiq
may canonicalize these.
*/
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
index bf57128..6ab7bbe 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
@@ -704,6 +704,19 @@ public class TestParquetFilterPushDown extends
PlanTestBase {
testParquetFilterPruning(query, 25, 1, new String[]{"Filter\\("});
}
+ @Test
+ public void testPreservingColumnAliasAfterRemovingFilter() throws Exception {
+ test("create table dfs.tmp.`testAliasTable` as select * from
cp.`tpch/nation.parquet` limit 1");
+ String query = "select n_regionkey as x from dfs.tmp.`testAliasTable`
where n_nationkey > -100500";
+
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("x")
+ .baselineValues(0)
+ .go();
+ }
+
//////////////////////////////////////////////////////////////////////////////////////////////////
// Some test helper functions.
//////////////////////////////////////////////////////////////////////////////////////////////////
