[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16577154#comment-16577154
]
ASF GitHub Bot commented on DRILL-6671:
---
asfgit closed pull request #1426: DRILL-6671: Multi level lateral unnest join
is throwing an exception …
URL: https://github.com/apache/drill/pull/1426
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java
index 987e65c2435..2f7fdce1839 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.planner.fragment;
+import java.util.ArrayDeque;
+import java.util.Deque;
import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
@@ -116,29 +118,28 @@ public PhysicalOperator visitLateralJoin(LateralJoinPOP
op, IndexedFragmentNode
children.add(op.getLeft().accept(this, iNode));
children.add(op.getRight().accept(this, iNode));
-UnnestPOP unnestInLeftInput = iNode.getUnnest();
+UnnestPOP unnestForThisLateral = iNode.getUnnest();
PhysicalOperator newOp = op.getNewWithChildren(children);
newOp.setCost(op.getCost());
newOp.setOperatorId(Short.MAX_VALUE & op.getOperatorId());
-((LateralJoinPOP)newOp).setUnnestForLateralJoin(unnestInLeftInput);
-
+((LateralJoinPOP) newOp).setUnnestForLateralJoin(unnestForThisLateral);
return newOp;
}
@Override
public PhysicalOperator visitUnnest(UnnestPOP unnest, IndexedFragmentNode
value) throws ExecutionSetupException {
PhysicalOperator newOp = visitOp(unnest, value);
-value.addUnnest((UnnestPOP)newOp);
+value.addUnnest((UnnestPOP) newOp);
return newOp;
}
public static class IndexedFragmentNode{
-final Wrapper info;
-final int minorFragmentId;
+private final Wrapper info;
+private final int minorFragmentId;
-UnnestPOP unnest = null;
+private final Deque unnest = new ArrayDeque<>();
public IndexedFragmentNode(int minorFragmentId, Wrapper info) {
super();
@@ -163,11 +164,11 @@ public void addAllocation(PhysicalOperator pop) {
}
public void addUnnest(UnnestPOP unnest) {
- this.unnest = unnest;
+ this.unnest.addFirst(unnest);
}
public UnnestPOP getUnnest() {
- return this.unnest;
+ return this.unnest.removeFirst();
}
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java
index 274f27a2e0b..a3449150918 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java
@@ -25,6 +25,7 @@
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.base.PhysicalOperator;
@@ -85,6 +86,15 @@ public boolean needsFinalColumnReordering() {
return LateralJoinPrel.class;
}
+ @Override
+ public RelNode accept(RexShuttle shuttle) {
+RexNode ref = shuttle.apply(this.ref);
+if (this.ref == ref) {
+ return this;
+}
+return new UnnestPrel(getCluster(), traitSet, rowType, ref);
+ }
+
@Override
public Prel prepareForLateralUnnestPipeline(List children) {
RelDataTypeFactory typeFactory = this.getCluster().getTypeFactory();
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
index dc4af5b08ff..7734d90aa16 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
@@ -19,11 +19,18 @@
import com.google.common.collect.Lists;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.CorrelationId;
+import org.apache.calcite.rel.rules.ProjectCorrelateTransposeRule;
+import org.apache.calcite.rex.RexCorrelVariable;
+import org.apa
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16576332#comment-16576332
]
ASF GitHub Bot commented on DRILL-6671:
---
HanumathRao commented on a change in pull request #1426: DRILL-6671: Multi
level lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r209274566
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
##
@@ -59,16 +66,55 @@ public Prel visitPrel(Prel prel, Boolean isRightOfLateral)
throws RuntimeExcepti
}
@Override
- public Prel visitLateral(LateralJoinPrel prel, Boolean value) throws
RuntimeException {
+ public Prel visitLateral(LateralJoinPrel prel, Boolean isRightOfLateral)
throws RuntimeException {
List children = Lists.newArrayList();
-children.add(((Prel)prel.getInput(0)).accept(this, value));
+children.add(((Prel) prel.getInput(0)).accept(this, isRightOfLateral));
children.add(((Prel) prel.getInput(1)).accept(this, true));
-return (Prel) prel.copy(prel.getTraitSet(), children);
+if (!isRightOfLateral) {
+ return (Prel) prel.copy(prel.getTraitSet(), children);
+} else {
+ //Adjust the column numbering due to an additional column
"$drill_implicit_field$" is added to the inputs.
+ Map requiredColsMap = new HashMap<>();
+ for (Integer corrColIndex : prel.getRequiredColumns()) {
+requiredColsMap.put(corrColIndex, corrColIndex + 1);
+ }
+ ImmutableBitSet requiredColumns = prel.getRequiredColumns().shift(1);
+
+ CorrelationId corrId = prel.getCluster().createCorrel();
+ RexCorrelVariable rexCorrel =
+ (RexCorrelVariable) prel.getCluster().getRexBuilder().makeCorrel(
+ children.get(0).getRowType(),
+ corrId);
+ RelNode rightChild = children.get(1).accept(
+ new CorrelateVarReplacer(
+ new
ProjectCorrelateTransposeRule.RexFieldAccessReplacer(corrId,
Review comment:
Done.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16575914#comment-16575914
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r209177260
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
##
@@ -59,16 +66,55 @@ public Prel visitPrel(Prel prel, Boolean isRightOfLateral)
throws RuntimeExcepti
}
@Override
- public Prel visitLateral(LateralJoinPrel prel, Boolean value) throws
RuntimeException {
+ public Prel visitLateral(LateralJoinPrel prel, Boolean isRightOfLateral)
throws RuntimeException {
List children = Lists.newArrayList();
-children.add(((Prel)prel.getInput(0)).accept(this, value));
+children.add(((Prel) prel.getInput(0)).accept(this, isRightOfLateral));
children.add(((Prel) prel.getInput(1)).accept(this, true));
-return (Prel) prel.copy(prel.getTraitSet(), children);
+if (!isRightOfLateral) {
+ return (Prel) prel.copy(prel.getTraitSet(), children);
+} else {
+ //Adjust the column numbering due to an additional column
"$drill_implicit_field$" is added to the inputs.
+ Map requiredColsMap = new HashMap<>();
+ for (Integer corrColIndex : prel.getRequiredColumns()) {
+requiredColsMap.put(corrColIndex, corrColIndex + 1);
+ }
+ ImmutableBitSet requiredColumns = prel.getRequiredColumns().shift(1);
+
+ CorrelationId corrId = prel.getCluster().createCorrel();
+ RexCorrelVariable rexCorrel =
+ (RexCorrelVariable) prel.getCluster().getRexBuilder().makeCorrel(
+ children.get(0).getRowType(),
+ corrId);
+ RelNode rightChild = children.get(1).accept(
+ new CorrelateVarReplacer(
+ new
ProjectCorrelateTransposeRule.RexFieldAccessReplacer(corrId,
Review comment:
The first argument in `RexFieldAccessReplacer` is a `CorrelationId` which
should be replaced, but you have specified the new one.
Please replace here `corrId` with `prel.getCorrelationId()`.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[ https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16575575#comment-16575575 ] ASF GitHub Bot commented on DRILL-6671: --- HanumathRao commented on issue #1426: DRILL-6671: Multi level lateral unnest join is throwing an exception … URL: https://github.com/apache/drill/pull/1426#issuecomment-411931640 @vvysotskyi I have made the changes. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > Multi level lateral unnest join is throwing an exception during materializing > the plan. > --- > > Key: DRILL-6671 > URL: https://issues.apache.org/jira/browse/DRILL-6671 > Project: Apache Drill > Issue Type: Bug > Components: Query Planning & Optimization >Affects Versions: 1.15.0 >Reporter: Hanumath Rao Maduri >Assignee: Hanumath Rao Maduri >Priority: Major > > testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an > execution in Materializer.java. This is due to incorrect matching of Unnest > and Lateral join. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16574528#comment-16574528
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208840867
##
File path:
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java
##
@@ -169,13 +169,31 @@ public void
testLeftLateral_WithFilterAndLimitInSubQuery() throws Exception {
@Test
public void testMultiUnnestAtSameLevel() throws Exception {
String Sql = "EXPLAIN PLAN FOR SELECT customer.c_name, customer.c_address,
U1.order_id, U1.order_amt," +
- " U1.itemName, U1.itemNum" + " FROM
cp.`lateraljoin/nested-customer.parquet` customer, LATERAL" +
- " (SELECT t.ord.o_id AS order_id, t.ord.o_amount AS order_amt,
U2.item_name AS itemName, U2.item_num AS " +
-"itemNum FROM UNNEST(customer.orders) t(ord) , LATERAL" +
- " (SELECT t1.ord.i_name AS item_name, t1.ord.i_number AS item_num FROM
UNNEST(t.ord) AS t1(ord)) AS U2) AS U1";
+" U1.itemName, U1.itemNum" + " FROM
cp.`lateraljoin/nested-customer.parquet` customer, LATERAL" +
+" (SELECT t.ord.o_id AS order_id, t.ord.o_amount AS order_amt,
U2.item_name AS itemName, U2.item_num AS " +
+"itemNum FROM UNNEST(customer.orders) t(ord) , LATERAL" +
+" (SELECT t1.ord.i_name AS item_name, t1.ord.i_number AS item_num
FROM UNNEST(t.ord) AS t1(ord)) AS U2) AS U1";
runAndLog(Sql);
}
+ @Test
+ public void testMultiUnnestAtSameLevelExec() throws Exception {
Review comment:
Could you please rewrite this test to validate query result with some
expected result, since it is possible that after some changes both these
queries may return the same incorrect result.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16574529#comment-16574529
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208856911
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
##
@@ -59,16 +65,37 @@ public Prel visitPrel(Prel prel, Boolean isRightOfLateral)
throws RuntimeExcepti
}
@Override
- public Prel visitLateral(LateralJoinPrel prel, Boolean value) throws
RuntimeException {
+ public Prel visitLateral(LateralJoinPrel prel, Boolean isRightOfLateral)
throws RuntimeException {
List children = Lists.newArrayList();
-children.add(((Prel)prel.getInput(0)).accept(this, value));
+children.add(((Prel)prel.getInput(0)).accept(this, isRightOfLateral));
children.add(((Prel) prel.getInput(1)).accept(this, true));
-return (Prel) prel.copy(prel.getTraitSet(), children);
+if (!isRightOfLateral) {
+ return (Prel) prel.copy(prel.getTraitSet(), children);
+} else {
+ //Adjust the column numbering due to an additional column
"$drill_implicit_field$" is added to the inputs.
+ Map requiredColsMap = new HashMap<>();
+ for (Integer corrColIndex : prel.getRequiredColumns()) {
+requiredColsMap.put(corrColIndex, corrColIndex+1);
+ }
+ ImmutableBitSet requiredColumns = prel.getRequiredColumns().shift(1);
+
+ CorrelationId corrId = prel.getCluster().createCorrel();
+ RexCorrelVariable rexCorrel =
+ (RexCorrelVariable) prel.getCluster().getRexBuilder().makeCorrel(
+ children.get(0).getRowType(),
+ corrId);
+ RelNode rightChild = children.get(1).accept(
Review comment:
Looks like `ProjectCorrelateTransposeRule.RelNodesExprsHandler` does not
replace RexCorrelVariable for the case when RelNode does not have any input. In
our case, this is `UnnestPrel`. So we need to extend this class and override
method `visit(RelNode other)`:
```
@Override
public RelNode visit(RelNode other) {
return super.visit(other.accept(rexVisitor));
}
```
Also, it does not work since `UnnestPrel` use `accept(RexShuttle shuttle)`
method from `AbstractRelNode` which does nothing, so we should override it
somehow like:
```
@Override
public RelNode accept(RexShuttle shuttle) {
RexNode ref = shuttle.apply(this.ref);
if (this.ref == ref) {
return this;
}
return new UnnestPrel(getCluster(), traitSet, rowType, ref);
}
```
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[ https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16574525#comment-16574525 ] ASF GitHub Bot commented on DRILL-6671: --- vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level lateral unnest join is throwing an exception … URL: https://github.com/apache/drill/pull/1426#discussion_r208839659 ## File path: exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java ## @@ -138,7 +139,7 @@ public PhysicalOperator visitUnnest(UnnestPOP unnest, IndexedFragmentNode value) final Wrapper info; final int minorFragmentId; Review comment: It would be good to make them private too. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > Multi level lateral unnest join is throwing an exception during materializing > the plan. > --- > > Key: DRILL-6671 > URL: https://issues.apache.org/jira/browse/DRILL-6671 > Project: Apache Drill > Issue Type: Bug > Components: Query Planning & Optimization >Affects Versions: 1.15.0 >Reporter: Hanumath Rao Maduri >Assignee: Hanumath Rao Maduri >Priority: Major > > testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an > execution in Materializer.java. This is due to incorrect matching of Unnest > and Lateral join. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16574530#comment-16574530
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208841312
##
File path:
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java
##
@@ -169,13 +169,31 @@ public void
testLeftLateral_WithFilterAndLimitInSubQuery() throws Exception {
@Test
public void testMultiUnnestAtSameLevel() throws Exception {
String Sql = "EXPLAIN PLAN FOR SELECT customer.c_name, customer.c_address,
U1.order_id, U1.order_amt," +
Review comment:
And here: `Sql` -> `sql`
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16574523#comment-16574523
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208841144
##
File path:
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java
##
@@ -169,13 +169,31 @@ public void
testLeftLateral_WithFilterAndLimitInSubQuery() throws Exception {
@Test
public void testMultiUnnestAtSameLevel() throws Exception {
String Sql = "EXPLAIN PLAN FOR SELECT customer.c_name, customer.c_address,
U1.order_id, U1.order_amt," +
- " U1.itemName, U1.itemNum" + " FROM
cp.`lateraljoin/nested-customer.parquet` customer, LATERAL" +
- " (SELECT t.ord.o_id AS order_id, t.ord.o_amount AS order_amt,
U2.item_name AS itemName, U2.item_num AS " +
-"itemNum FROM UNNEST(customer.orders) t(ord) , LATERAL" +
- " (SELECT t1.ord.i_name AS item_name, t1.ord.i_number AS item_num FROM
UNNEST(t.ord) AS t1(ord)) AS U2) AS U1";
+" U1.itemName, U1.itemNum" + " FROM
cp.`lateraljoin/nested-customer.parquet` customer, LATERAL" +
+" (SELECT t.ord.o_id AS order_id, t.ord.o_amount AS order_amt,
U2.item_name AS itemName, U2.item_num AS " +
+"itemNum FROM UNNEST(customer.orders) t(ord) , LATERAL" +
+" (SELECT t1.ord.i_name AS item_name, t1.ord.i_number AS item_num
FROM UNNEST(t.ord) AS t1(ord)) AS U2) AS U1";
runAndLog(Sql);
}
+ @Test
+ public void testMultiUnnestAtSameLevelExec() throws Exception {
+String Sql = "SELECT customer.c_name, customer.c_address, U1.order_id,
U1.order_amt," +
Review comment:
Please rename string in lower case: `Sql` -> `sql`
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[ https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16574524#comment-16574524 ] ASF GitHub Bot commented on DRILL-6671: --- vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level lateral unnest join is throwing an exception … URL: https://github.com/apache/drill/pull/1426#discussion_r208839528 ## File path: exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java ## @@ -116,14 +118,13 @@ public PhysicalOperator visitLateralJoin(LateralJoinPOP op, IndexedFragmentNode children.add(op.getLeft().accept(this, iNode)); children.add(op.getRight().accept(this, iNode)); -UnnestPOP unnestInLeftInput = iNode.getUnnest(); +UnnestPOP unnestForThisLateral = iNode.getUnnest(); PhysicalOperator newOp = op.getNewWithChildren(children); newOp.setCost(op.getCost()); newOp.setOperatorId(Short.MAX_VALUE & op.getOperatorId()); -((LateralJoinPOP)newOp).setUnnestForLateralJoin(unnestInLeftInput); - +((LateralJoinPOP)newOp).setUnnestForLateralJoin(unnestForThisLateral); Review comment: Please add space: `((LateralJoinPOP)newOp)` -> `((LateralJoinPOP) newOp)` This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > Multi level lateral unnest join is throwing an exception during materializing > the plan. > --- > > Key: DRILL-6671 > URL: https://issues.apache.org/jira/browse/DRILL-6671 > Project: Apache Drill > Issue Type: Bug > Components: Query Planning & Optimization >Affects Versions: 1.15.0 >Reporter: Hanumath Rao Maduri >Assignee: Hanumath Rao Maduri >Priority: Major > > testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an > execution in Materializer.java. This is due to incorrect matching of Unnest > and Lateral join. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16574527#comment-16574527
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208839935
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
##
@@ -59,16 +65,37 @@ public Prel visitPrel(Prel prel, Boolean isRightOfLateral)
throws RuntimeExcepti
}
@Override
- public Prel visitLateral(LateralJoinPrel prel, Boolean value) throws
RuntimeException {
+ public Prel visitLateral(LateralJoinPrel prel, Boolean isRightOfLateral)
throws RuntimeException {
List children = Lists.newArrayList();
-children.add(((Prel)prel.getInput(0)).accept(this, value));
+children.add(((Prel)prel.getInput(0)).accept(this, isRightOfLateral));
Review comment:
Please add space here and in the line below: `(Prel)prel` -> `(Prel) prel`
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16574526#comment-16574526
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208840141
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
##
@@ -59,16 +65,37 @@ public Prel visitPrel(Prel prel, Boolean isRightOfLateral)
throws RuntimeExcepti
}
@Override
- public Prel visitLateral(LateralJoinPrel prel, Boolean value) throws
RuntimeException {
+ public Prel visitLateral(LateralJoinPrel prel, Boolean isRightOfLateral)
throws RuntimeException {
List children = Lists.newArrayList();
-children.add(((Prel)prel.getInput(0)).accept(this, value));
+children.add(((Prel)prel.getInput(0)).accept(this, isRightOfLateral));
children.add(((Prel) prel.getInput(1)).accept(this, true));
-return (Prel) prel.copy(prel.getTraitSet(), children);
+if (!isRightOfLateral) {
+ return (Prel) prel.copy(prel.getTraitSet(), children);
+} else {
+ //Adjust the column numbering due to an additional column
"$drill_implicit_field$" is added to the inputs.
+ Map requiredColsMap = new HashMap<>();
+ for (Integer corrColIndex : prel.getRequiredColumns()) {
+requiredColsMap.put(corrColIndex, corrColIndex+1);
Review comment:
Please add spaces: `corrColIndex+1` -> `corrColIndex + 1`
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16573651#comment-16573651
]
ASF GitHub Bot commented on DRILL-6671:
---
HanumathRao commented on a change in pull request #1426: DRILL-6671: Multi
level lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208685651
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
##
@@ -64,7 +66,14 @@ public Prel visitLateral(LateralJoinPrel prel, Boolean
value) throws RuntimeExce
children.add(((Prel)prel.getInput(0)).accept(this, value));
children.add(((Prel) prel.getInput(1)).accept(this, true));
-return (Prel) prel.copy(prel.getTraitSet(), children);
+if (!value) {
+ return (Prel) prel.copy(prel.getTraitSet(), children);
+} else {
+ CorrelationId corrId = new CorrelationId(prel.getCorrelationId().getId()
+ 1);
Review comment:
As discussed offline. I have used the same code as that of
ProjectCorrelateTransposeRule to replace correlate variables.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[ https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16573655#comment-16573655 ] ASF GitHub Bot commented on DRILL-6671: --- HanumathRao commented on issue #1426: DRILL-6671: Multi level lateral unnest join is throwing an exception … URL: https://github.com/apache/drill/pull/1426#issuecomment-411504084 @vvysotskyi Thank you for the review. I have made the changes. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > Multi level lateral unnest join is throwing an exception during materializing > the plan. > --- > > Key: DRILL-6671 > URL: https://issues.apache.org/jira/browse/DRILL-6671 > Project: Apache Drill > Issue Type: Bug > Components: Query Planning & Optimization >Affects Versions: 1.15.0 >Reporter: Hanumath Rao Maduri >Assignee: Hanumath Rao Maduri >Priority: Major > > testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an > execution in Materializer.java. This is due to incorrect matching of Unnest > and Lateral join. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16573650#comment-16573650
]
ASF GitHub Bot commented on DRILL-6671:
---
HanumathRao commented on a change in pull request #1426: DRILL-6671: Multi
level lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208685636
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
##
@@ -64,7 +66,14 @@ public Prel visitLateral(LateralJoinPrel prel, Boolean
value) throws RuntimeExce
children.add(((Prel)prel.getInput(0)).accept(this, value));
children.add(((Prel) prel.getInput(1)).accept(this, true));
-return (Prel) prel.copy(prel.getTraitSet(), children);
+if (!value) {
Review comment:
renamed it.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16573652#comment-16573652
]
ASF GitHub Bot commented on DRILL-6671:
---
HanumathRao commented on a change in pull request #1426: DRILL-6671: Multi
level lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208685665
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
##
@@ -64,7 +66,14 @@ public Prel visitLateral(LateralJoinPrel prel, Boolean
value) throws RuntimeExce
children.add(((Prel)prel.getInput(0)).accept(this, value));
children.add(((Prel) prel.getInput(1)).accept(this, true));
-return (Prel) prel.copy(prel.getTraitSet(), children);
+if (!value) {
+ return (Prel) prel.copy(prel.getTraitSet(), children);
+} else {
+ CorrelationId corrId = new CorrelationId(prel.getCorrelationId().getId()
+ 1);
+ ImmutableBitSet requiredColumns = prel.getRequiredColumns().shift(1);
Review comment:
Done.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[ https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16573648#comment-16573648 ] ASF GitHub Bot commented on DRILL-6671: --- HanumathRao commented on a change in pull request #1426: DRILL-6671: Multi level lateral unnest join is throwing an exception … URL: https://github.com/apache/drill/pull/1426#discussion_r208685608 ## File path: exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java ## @@ -138,7 +138,7 @@ public PhysicalOperator visitUnnest(UnnestPOP unnest, IndexedFragmentNode value) final Wrapper info; final int minorFragmentId; -UnnestPOP unnest = null; +Stack unnest = new Stack<>(); Review comment: Done. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > Multi level lateral unnest join is throwing an exception during materializing > the plan. > --- > > Key: DRILL-6671 > URL: https://issues.apache.org/jira/browse/DRILL-6671 > Project: Apache Drill > Issue Type: Bug > Components: Query Planning & Optimization >Affects Versions: 1.15.0 >Reporter: Hanumath Rao Maduri >Assignee: Hanumath Rao Maduri >Priority: Major > > testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an > execution in Materializer.java. This is due to incorrect matching of Unnest > and Lateral join. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16573649#comment-16573649
]
ASF GitHub Bot commented on DRILL-6671:
---
HanumathRao commented on a change in pull request #1426: DRILL-6671: Multi
level lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208685624
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java
##
@@ -48,7 +48,7 @@
public class LateralJoinPrel extends DrillLateralJoinRelBase implements Prel {
- protected LateralJoinPrel(RelOptCluster cluster, RelTraitSet traits, RelNode
left, RelNode right, boolean excludeCorrelateCol,
+ public LateralJoinPrel(RelOptCluster cluster, RelTraitSet traits, RelNode
left, RelNode right, boolean excludeCorrelateCol,
Review comment:
Reverted it back.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16573459#comment-16573459
]
ASF GitHub Bot commented on DRILL-6671:
---
HanumathRao commented on a change in pull request #1426: DRILL-6671: Multi
level lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208649747
##
File path:
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java
##
@@ -169,13 +169,31 @@ public void
testLeftLateral_WithFilterAndLimitInSubQuery() throws Exception {
@Test
public void testMultiUnnestAtSameLevel() throws Exception {
String Sql = "EXPLAIN PLAN FOR SELECT customer.c_name, customer.c_address,
U1.order_id, U1.order_amt," +
Review comment:
@vvysotskyi This is an existing testcase. I just want to keep it intact. I
cannot remove the EXPLAIN PLAN and use it for executing the query because this
query will result in an execution error (unnest is not supported for non list
types) and this is the right behaviour. For testing the usecase of this PR an
additional testcase has been introduced which also tests for the correctness of
the result.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16572956#comment-16572956
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208516883
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
##
@@ -64,7 +66,14 @@ public Prel visitLateral(LateralJoinPrel prel, Boolean
value) throws RuntimeExce
children.add(((Prel)prel.getInput(0)).accept(this, value));
children.add(((Prel) prel.getInput(1)).accept(this, true));
-return (Prel) prel.copy(prel.getTraitSet(), children);
+if (!value) {
Review comment:
As I understand, `value` argument is used to specify that the current `prel`
is the right input of the `LateralJoinPrel` if it is true, I recommend to
rename it for better readability.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[ https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16572954#comment-16572954 ] ASF GitHub Bot commented on DRILL-6671: --- vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level lateral unnest join is throwing an exception … URL: https://github.com/apache/drill/pull/1426#discussion_r208514180 ## File path: exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java ## @@ -138,7 +138,7 @@ public PhysicalOperator visitUnnest(UnnestPOP unnest, IndexedFragmentNode value) final Wrapper info; final int minorFragmentId; -UnnestPOP unnest = null; +Stack unnest = new Stack<>(); Review comment: Could you please replace `Stack` usage by `Deque` and `ArrayDeque` and make fields private final This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > Multi level lateral unnest join is throwing an exception during materializing > the plan. > --- > > Key: DRILL-6671 > URL: https://issues.apache.org/jira/browse/DRILL-6671 > Project: Apache Drill > Issue Type: Bug > Components: Query Planning & Optimization >Affects Versions: 1.15.0 >Reporter: Hanumath Rao Maduri >Assignee: Hanumath Rao Maduri >Priority: Major > > testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an > execution in Materializer.java. This is due to incorrect matching of Unnest > and Lateral join. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16572953#comment-16572953
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208521406
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
##
@@ -64,7 +66,14 @@ public Prel visitLateral(LateralJoinPrel prel, Boolean
value) throws RuntimeExce
children.add(((Prel)prel.getInput(0)).accept(this, value));
children.add(((Prel) prel.getInput(1)).accept(this, true));
-return (Prel) prel.copy(prel.getTraitSet(), children);
+if (!value) {
+ return (Prel) prel.copy(prel.getTraitSet(), children);
+} else {
+ CorrelationId corrId = new CorrelationId(prel.getCorrelationId().getId()
+ 1);
+ ImmutableBitSet requiredColumns = prel.getRequiredColumns().shift(1);
Review comment:
Could you please add a comment that shift is required for taking into
account newly added `$drill_implicit_field$`
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16572957#comment-16572957
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208515217
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java
##
@@ -48,7 +48,7 @@
public class LateralJoinPrel extends DrillLateralJoinRelBase implements Prel {
- protected LateralJoinPrel(RelOptCluster cluster, RelTraitSet traits, RelNode
left, RelNode right, boolean excludeCorrelateCol,
+ public LateralJoinPrel(RelOptCluster cluster, RelTraitSet traits, RelNode
left, RelNode right, boolean excludeCorrelateCol,
Review comment:
I think there is no need to make this constructor public. In
`LateralUnnestRowIDVisitor` may be used method `copy()` instead of using the
constructor.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16572955#comment-16572955
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208509635
##
File path:
exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java
##
@@ -169,13 +169,31 @@ public void
testLeftLateral_WithFilterAndLimitInSubQuery() throws Exception {
@Test
public void testMultiUnnestAtSameLevel() throws Exception {
String Sql = "EXPLAIN PLAN FOR SELECT customer.c_name, customer.c_address,
U1.order_id, U1.order_amt," +
Review comment:
If this fix allows execution for the query from the test, I think we should
remove `EXPLAIN PLAN FOR` and check the result.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[
https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16572958#comment-16572958
]
ASF GitHub Bot commented on DRILL-6671:
---
vvysotskyi commented on a change in pull request #1426: DRILL-6671: Multi level
lateral unnest join is throwing an exception …
URL: https://github.com/apache/drill/pull/1426#discussion_r208520804
##
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/LateralUnnestRowIDVisitor.java
##
@@ -64,7 +66,14 @@ public Prel visitLateral(LateralJoinPrel prel, Boolean
value) throws RuntimeExce
children.add(((Prel)prel.getInput(0)).accept(this, value));
children.add(((Prel) prel.getInput(1)).accept(this, true));
-return (Prel) prel.copy(prel.getTraitSet(), children);
+if (!value) {
+ return (Prel) prel.copy(prel.getTraitSet(), children);
+} else {
+ CorrelationId corrId = new CorrelationId(prel.getCorrelationId().getId()
+ 1);
Review comment:
Passing increased `id` to `CorrelationId` may cause problems since inputs of
`LateralJoinPrel` also may contain both `CorrelationId`s with old and new `id`s
and it may cause problems for some queries. I would recommend receiving
`CorrelationId` by calling `prel.getCluster().createCorrel()`, since it
guarantees that the index will be unique.
Also, just creating and passing a new `CorrelationId` instance to the
`LateralJoinPrel` is not enough, because as I mentioned above, inputs refer to
it, so all these references should be replaced. As an example of this approach,
you may use code from
[ProjectCorrelateTransposeRule](https://github.com/apache/calcite/blob/master/core/src/main/java/org/apache/calcite/rel/rules/ProjectCorrelateTransposeRule.java#L124).
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Multi level lateral unnest join is throwing an exception during materializing
> the plan.
> ---
>
> Key: DRILL-6671
> URL: https://issues.apache.org/jira/browse/DRILL-6671
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
>Affects Versions: 1.15.0
>Reporter: Hanumath Rao Maduri
>Assignee: Hanumath Rao Maduri
>Priority: Major
>
> testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an
> execution in Materializer.java. This is due to incorrect matching of Unnest
> and Lateral join.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[ https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16572515#comment-16572515 ] ASF GitHub Bot commented on DRILL-6671: --- HanumathRao commented on issue #1426: DRILL-6671: Multi level lateral unnest join is throwing an exception … URL: https://github.com/apache/drill/pull/1426#issuecomment-411245012 @vvysotskyi Please can you help review this change. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > Multi level lateral unnest join is throwing an exception during materializing > the plan. > --- > > Key: DRILL-6671 > URL: https://issues.apache.org/jira/browse/DRILL-6671 > Project: Apache Drill > Issue Type: Bug > Components: Query Planning & Optimization >Affects Versions: 1.15.0 >Reporter: Hanumath Rao Maduri >Assignee: Hanumath Rao Maduri >Priority: Major > > testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an > execution in Materializer.java. This is due to incorrect matching of Unnest > and Lateral join. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Commented] (DRILL-6671) Multi level lateral unnest join is throwing an exception during materializing the plan.
[ https://issues.apache.org/jira/browse/DRILL-6671?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16572451#comment-16572451 ] ASF GitHub Bot commented on DRILL-6671: --- HanumathRao opened a new pull request #1426: DRILL-6671: Multi level lateral unnest join is throwing an exception … URL: https://github.com/apache/drill/pull/1426 …during materializing the plan. This PR includes changes to correctly attach the concerned UnnestPOP to LateralJoinPOP. The logic is to push the UnnestPOP as we are visiting the lateral join nodes and pop the unnest pop and attach it to the lateraljoin. @gparai @sohami Can you please review this change. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > Multi level lateral unnest join is throwing an exception during materializing > the plan. > --- > > Key: DRILL-6671 > URL: https://issues.apache.org/jira/browse/DRILL-6671 > Project: Apache Drill > Issue Type: Bug > Components: Query Planning & Optimization >Affects Versions: 1.15.0 >Reporter: Hanumath Rao Maduri >Assignee: Hanumath Rao Maduri >Priority: Major > > testMultiUnnestAtSameLevel in TestE2EUnnestAndLateral is throwing an > execution in Materializer.java. This is due to incorrect matching of Unnest > and Lateral join. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
