This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push: new cacca92 DRILL-6546: Allow unnest function with nested columns and complex expressions cacca92 is described below commit cacca92fde38208828fea71d449ebb67ad9fc10f Author: Volodymyr Vysotskyi <vvo...@gmail.com> AuthorDate: Thu Jun 14 19:32:43 2018 +0300 DRILL-6546: Allow unnest function with nested columns and complex expressions Fix loss of projected names in right side of correlate when single field is projected --- .../apache/drill/exec/planner/PlannerPhase.java | 3 + .../planner/common/DrillLateralJoinRelBase.java | 19 ++- .../exec/planner/logical/DrillLateralJoinRel.java | 2 +- .../exec/planner/logical/DrillUnnestRule.java | 11 +- ...rojectComplexRexNodeCorrelateTransposeRule.java | 154 +++++++++++++++++++ .../exec/planner/physical/LateralJoinPrel.java | 5 +- .../drill/exec/planner/physical/UnnestPrule.java | 5 - .../physical/visitor/JoinPrelRenameVisitor.java | 14 +- .../sql/parser/CompoundIdentifierConverter.java | 171 +++++++++++---------- .../impl/lateraljoin/TestE2EUnnestAndLateral.java | 88 +++++++++++ 10 files changed, 362 insertions(+), 110 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java index 519d503..e5a3746 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java @@ -37,6 +37,7 @@ import org.apache.drill.exec.planner.logical.DrillJoinRel; import org.apache.drill.exec.planner.logical.DrillJoinRule; import org.apache.drill.exec.planner.logical.DrillLimitRule; import org.apache.drill.exec.planner.logical.DrillMergeProjectRule; +import org.apache.drill.exec.planner.logical.ProjectComplexRexNodeCorrelateTransposeRule; import org.apache.drill.exec.planner.logical.DrillProjectLateralJoinTransposeRule; import org.apache.drill.exec.planner.logical.DrillProjectPushIntoLateralJoinRule; import org.apache.drill.exec.planner.logical.DrillProjectRule; @@ -311,6 +312,8 @@ public enum PlannerPhase { RuleInstance.PROJECT_WINDOW_TRANSPOSE_RULE, DrillPushProjectIntoScanRule.INSTANCE, + ProjectComplexRexNodeCorrelateTransposeRule.INSTANCE, + /* Convert from Calcite Logical to Drill Logical Rules. */ diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java index 28e5246..2f895e2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillLateralJoinRelBase.java @@ -73,7 +73,7 @@ public abstract class DrillLateralJoinRelBase extends Correlate implements Drill return constructRowType(SqlValidatorUtil.deriveJoinRowType(left.getRowType(), right.getRowType(), joinType.toJoinType(), getCluster().getTypeFactory(), null, - ImmutableList.<RelDataTypeField>of())); + ImmutableList.of())); case ANTI: case SEMI: return constructRowType(left.getRowType()); @@ -82,12 +82,19 @@ public abstract class DrillLateralJoinRelBase extends Correlate implements Drill } } - public int getInputSize(int offset, RelNode input) { - if (this.excludeCorrelateColumn && - offset == 0) { - return input.getRowType().getFieldList().size() - 1; + /** + * Returns number of fields in {@link RelDataType} for + * input rel node with specified ordinal considering value of + * {@code excludeCorrelateColumn}. + * + * @param ordinal ordinal of input rel node + * @return number of fields in input's {@link RelDataType} + */ + public int getInputSize(int ordinal) { + if (this.excludeCorrelateColumn && ordinal == 0) { + return getInput(ordinal).getRowType().getFieldList().size() - 1; } - return input.getRowType().getFieldList().size(); + return getInput(ordinal).getRowType().getFieldList().size(); } public RelDataType constructRowType(RelDataType inputRowType) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java index aa6ccb0..4356d49 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillLateralJoinRel.java @@ -50,7 +50,7 @@ public class DrillLateralJoinRel extends DrillLateralJoinRelBase implements Dril public LogicalOperator implement(DrillImplementor implementor) { final List<String> fields = getRowType().getFieldNames(); assert DrillJoinRel.isUnique(fields); - final int leftCount = getInputSize(0,left); + final int leftCount = getInputSize(0); final LogicalOperator leftOp = DrillJoinRel.implementInput(implementor, 0, 0, left, this); final LogicalOperator rightOp = DrillJoinRel.implementInput(implementor, 1, leftCount, right, this); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnnestRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnnestRule.java index 762eb46..ce0cd3c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnnestRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnnestRule.java @@ -24,6 +24,8 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.core.Uncollect; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.logical.LogicalValues; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; public class DrillUnnestRule extends RelOptRule { public static final RelOptRule INSTANCE = new DrillUnnestRule(); @@ -38,11 +40,14 @@ public class DrillUnnestRule extends RelOptRule { public void onMatch(RelOptRuleCall call) { final Uncollect uncollect = call.rel(0); final LogicalProject project = call.rel(1); - final LogicalValues values = call.rel(2); + RexNode projectedNode = project.getProjects().iterator().next(); + if (projectedNode.getKind() != SqlKind.FIELD_ACCESS) { + return; + } final RelTraitSet traits = uncollect.getTraitSet().plus(DrillRel.DRILL_LOGICAL); - DrillUnnestRel unnest = new DrillUnnestRel(uncollect.getCluster(), traits, uncollect.getRowType(), - project.getProjects().iterator().next()); + DrillUnnestRel unnest = new DrillUnnestRel(uncollect.getCluster(), + traits, uncollect.getRowType(), projectedNode); call.transformTo(unnest); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ProjectComplexRexNodeCorrelateTransposeRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ProjectComplexRexNodeCorrelateTransposeRule.java new file mode 100644 index 0000000..a979d5b --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ProjectComplexRexNodeCorrelateTransposeRule.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.logical; + +import com.google.common.collect.ImmutableList; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Correlate; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.logical.LogicalCorrelate; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.trace.CalciteTrace; +import org.apache.drill.common.exceptions.UserException; + +import java.util.ArrayList; +import java.util.List; + +/** + * Rule that moves non-{@link RexFieldAccess} rex node from project below {@link Uncollect} + * to the left side of the {@link Correlate}. + */ +public class ProjectComplexRexNodeCorrelateTransposeRule extends RelOptRule { + + public static final RelOptRule INSTANCE = new ProjectComplexRexNodeCorrelateTransposeRule(); + + public ProjectComplexRexNodeCorrelateTransposeRule() { + super(operand(LogicalCorrelate.class, + operand(RelNode.class, any()), + operand(Uncollect.class, operand(LogicalProject.class, any()))), + DrillRelFactories.LOGICAL_BUILDER, + "ProjectComplexRexNodeCorrelateTransposeRule"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final Correlate correlate = call.rel(0); + final Uncollect uncollect = call.rel(2); + final LogicalProject project = call.rel(3); + + // uncollect requires project with single expression + RexNode projectedNode = project.getProjects().iterator().next(); + + // check that the expression is complex call + if (!(projectedNode instanceof RexFieldAccess)) { + RelBuilder builder = call.builder(); + RexBuilder rexBuilder = builder.getRexBuilder(); + + builder.push(correlate.getLeft()); + + // creates project with complex expr on top of the left side + List<RexNode> leftProjExprs = new ArrayList<>(); + + String complexFieldName = correlate.getRowType().getFieldNames() + .get(correlate.getRowType().getFieldNames().size() - 1); + + List<String> fieldNames = new ArrayList<>(); + for (RelDataTypeField field : correlate.getLeft().getRowType().getFieldList()) { + leftProjExprs.add(rexBuilder.makeInputRef(correlate.getLeft(), field.getIndex())); + fieldNames.add(field.getName()); + } + fieldNames.add(complexFieldName); + List<RexNode> topProjectExpressions = new ArrayList<>(leftProjExprs); + + // adds complex expression with replaced correlation + // to the projected list from the left + leftProjExprs.add(projectedNode.accept(new RexFieldAccessReplacer(builder))); + + RelNode leftProject = builder.project(leftProjExprs, fieldNames) + .build(); + + CorrelationId correlationId = correlate.getCluster().createCorrel(); + RexCorrelVariable rexCorrel = + (RexCorrelVariable) rexBuilder.makeCorrel( + leftProject.getRowType(), + correlationId); + builder.push(project.getInput()); + RelNode rightProject = builder.project( + ImmutableList.of(rexBuilder.makeFieldAccess(rexCorrel, leftProjExprs.size() - 1)), + ImmutableList.of(complexFieldName)) + .build(); + + int requiredColumnsCount = correlate.getRequiredColumns().cardinality(); + if (requiredColumnsCount != 1) { + throw UserException.planError() + .message("Required columns count for Correlate operator " + + "differs from the expected value:\n" + + "Expected columns count is %s, but actual is %s", + 1, requiredColumnsCount) + .build(CalciteTrace.getPlannerTracer()); + } + + RelNode newUncollect = uncollect.copy(uncollect.getTraitSet(), rightProject); + Correlate newCorrelate = correlate.copy(uncollect.getTraitSet(), leftProject, newUncollect, + correlationId, ImmutableBitSet.of(leftProjExprs.size() - 1), correlate.getJoinType()); + builder.push(newCorrelate); + + switch(correlate.getJoinType()) { + case LEFT: + case INNER: + // adds field from the right input of correlate to the top project + topProjectExpressions.add( + rexBuilder.makeInputRef(newCorrelate, topProjectExpressions.size() + 1)); + // fall through + case ANTI: + case SEMI: + builder.project(topProjectExpressions, correlate.getRowType().getFieldNames()); + } + + call.transformTo(builder.build()); + } + } + + /** + * Visitor for RexNode which replaces {@link RexFieldAccess} + * with a reference to the field used in {@link RexFieldAccess}. + */ + private static class RexFieldAccessReplacer extends RexShuttle { + private final RelBuilder builder; + + public RexFieldAccessReplacer(RelBuilder builder) { + this.builder = builder; + } + + @Override + public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { + return builder.field(fieldAccess.getField().getName()); + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java index b55076b..b10eff0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LateralJoinPrel.java @@ -88,11 +88,12 @@ public class LateralJoinPrel extends DrillLateralJoinRelBase implements Prel { * Check to make sure that the fields of the inputs are the same as the output field names. * If not, insert a project renaming them. */ - public RelNode getLateralInput(int offset, RelNode input) { + public RelNode getLateralInput(int ordinal, RelNode input) { + int offset = ordinal == 0 ? 0 : getInputSize(0); Preconditions.checkArgument(DrillJoinRelBase.uniqueFieldNames(input.getRowType())); final List<String> fields = getRowType().getFieldNames(); final List<String> inputFields = input.getRowType().getFieldNames(); - final List<String> outputFields = fields.subList(offset, offset + getInputSize(offset, input)); + final List<String> outputFields = fields.subList(offset, offset + getInputSize(ordinal)); if (ListUtils.subtract(outputFields, inputFields).size() != 0) { // Ensure that input field names are the same as output field names. // If there are duplicate field names on left and right, fields will get diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrule.java index 48f4ea9..544a628 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrule.java @@ -19,7 +19,6 @@ package org.apache.drill.exec.planner.physical; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexNode; import org.apache.drill.exec.planner.logical.DrillUnnestRel; import org.apache.drill.exec.planner.logical.RelOptHelper; @@ -34,10 +33,6 @@ public class UnnestPrule extends Prule { public void onMatch(RelOptRuleCall call) { final DrillUnnestRel unnest = call.rel(0); RexNode ref = unnest.getRef(); - if (ref instanceof RexFieldAccess) { - final RexFieldAccess field = (RexFieldAccess)ref; - field.getField().getName(); - } UnnestPrel unnestPrel = new UnnestPrel(unnest.getCluster(), unnest.getTraitSet().plus(Prel.DRILL_PHYSICAL), unnest.getRowType(), ref); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java index 850f0bd..3a2529b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.planner.physical.visitor; +import java.util.ArrayList; import java.util.List; import org.apache.drill.exec.planner.physical.JoinPrel; @@ -75,16 +76,11 @@ public class JoinPrelRenameVisitor extends BasePrelVisitor<Prel, Void, RuntimeEx public Prel visitLateral(LateralJoinPrel prel, Void value) throws RuntimeException { List<RelNode> children = getChildren(prel); + List<RelNode> reNamedChildren = new ArrayList<>(); - final int leftCount = prel.getInputSize(0,children.get(0)); - - List<RelNode> reNamedChildren = Lists.newArrayList(); - - RelNode left = prel.getLateralInput(0, children.get(0)); - RelNode right = prel.getLateralInput(leftCount, children.get(1)); - - reNamedChildren.add(left); - reNamedChildren.add(right); + for (int i = 0; i < children.size(); i++) { + reNamedChildren.add(prel.getLateralInput(i, children.get(i))); + } return preparePrel(prel, reNamedChildren); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java index 4d0f34c..ded85c5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/CompoundIdentifierConverter.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; @@ -31,30 +32,68 @@ import org.apache.calcite.sql.util.SqlShuttle; import org.apache.calcite.sql.util.SqlVisitor; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; /** - * Implementation of {@link SqlVisitor} that converts bracketed compound {@link SqlIdentifier} to bracket-less compound - * {@link SqlIdentifier} (also known as {@link DrillCompoundIdentifier}) to provide ease of use while querying complex - * types. + * Implementation of {@link SqlVisitor} that converts bracketed compound {@link SqlIdentifier} + * to bracket-less compound {@link SqlIdentifier} (also known as {@link DrillCompoundIdentifier}) + * to provide ease of use while querying complex types. * <p/> * For example, this visitor converts {@code a['b'][4]['c']} to {@code a.b[4].c} */ public class CompoundIdentifierConverter extends SqlShuttle { -// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CompoundIdentifierConverter.class); + /** + * This map stores the rules that instruct each SqlCall class which data field needs + * to be rewritten if that data field is a {@link DrillCompoundIdentifier}. + * <p/> + * <ul> + * <li>Key : Each rule corresponds to a {@link SqlCall} class; + * <li>Value: It is an array of {@link RewriteType}, each being associated with a data field + * in that class. + * </ul> + * <p/> + * For example, there are four data fields (query, orderList, offset, fetch) + * in {@link SqlOrderBy}. Since only orderList needs to be written, + * {@link RewriteType[]} should be {@code arrayOf(D, E, D, D)}. + */ + private static final Map<Class<? extends SqlCall>, RewriteType[]> REWRITE_RULES; + + static { + final RewriteType E = RewriteType.ENABLE; + final RewriteType D = RewriteType.DISABLE; + + // Every element of the array corresponds to the item in the list + // returned by getOperandList() method for concrete SqlCall implementation. + REWRITE_RULES = ImmutableMap.<Class<? extends SqlCall>, RewriteType[]>builder() + .put(SqlSelect.class, arrayOf(D, E, D, E, E, E, E, E, D, D)) + .put(SqlCreateTable.class, arrayOf(D, D, D, E, D, D)) + .put(SqlCreateView.class, arrayOf(D, E, E, D)) + .put(DrillSqlDescribeTable.class, arrayOf(D, D, E)) + .put(SqlDropView.class, arrayOf(D, D)) + .put(SqlShowFiles.class, arrayOf(D)) + .put(SqlShowSchemas.class, arrayOf(D, D)) + .put(SqlUseSchema.class, arrayOf(D)) + .put(SqlJoin.class, arrayOf(D, D, D, D, D, E)) + .put(SqlOrderBy.class, arrayOf(D, E, D, D)) + .put(SqlDropTable.class, arrayOf(D, D)) + .put(SqlRefreshMetadata.class, arrayOf(D)) + .put(SqlSetOption.class, arrayOf(D, D, D)) + .put(SqlCreateFunction.class, arrayOf(D)) + .put(SqlDropFunction.class, arrayOf(D)) + .build(); + } private boolean enableComplex = true; @Override public SqlNode visit(SqlIdentifier id) { - if(id instanceof DrillCompoundIdentifier){ - if(enableComplex){ - return ((DrillCompoundIdentifier) id).getAsSqlNode(); - }else{ - return ((DrillCompoundIdentifier) id).getAsCompoundIdentifier(); + if (id instanceof DrillCompoundIdentifier) { + DrillCompoundIdentifier compoundIdentifier = (DrillCompoundIdentifier) id; + if (enableComplex) { + return compoundIdentifier.getAsSqlNode(); + } else { + return compoundIdentifier.getAsCompoundIdentifier(); } - - }else{ + } else { return id; } } @@ -64,22 +103,46 @@ public class CompoundIdentifierConverter extends SqlShuttle { // Handler creates a new copy of 'call' only if one or more operands // change. ArgHandler<SqlNode> argHandler = new ComplexExpressionAware(call); + boolean localEnableComplex = enableComplex; + // for the case of UNNEST call set enableComplex to true + // to convert DrillCompoundIdentifier to the item call. + if (call.getKind() == SqlKind.UNNEST) { + enableComplex = true; + } call.getOperator().acceptCall(this, call, false, argHandler); + enableComplex = localEnableComplex; return argHandler.result(); } + /** + * Constructs array which contains specified parameters. + */ + private static RewriteType[] arrayOf(RewriteType... types) { + return types; + } + + enum RewriteType { + UNCHANGED, DISABLE, ENABLE + } + + /** + * Argument handler which accepts {@link CompoundIdentifierConverter} + * for every operand of {@link SqlCall} and constructs new {@link SqlCall} + * if one or more operands changed. + */ + private class ComplexExpressionAware implements ArgHandler<SqlNode> { - private class ComplexExpressionAware implements ArgHandler<SqlNode> { - boolean update; - SqlNode[] clonedOperands; - RewriteType[] rewriteTypes; private final SqlCall call; + private final SqlNode[] clonedOperands; + private final RewriteType[] rewriteTypes; + + private boolean update; public ComplexExpressionAware(SqlCall call) { this.call = call; this.update = false; final List<SqlNode> operands = call.getOperandList(); - this.clonedOperands = operands.toArray(new SqlNode[operands.size()]); + this.clonedOperands = operands.toArray(new SqlNode[0]); rewriteTypes = REWRITE_RULES.get(call.getClass()); } @@ -106,13 +169,13 @@ public class CompoundIdentifierConverter extends SqlShuttle { } boolean localEnableComplex = enableComplex; - if(rewriteTypes != null){ - switch(rewriteTypes[i]){ - case DISABLE: - enableComplex = false; - break; - case ENABLE: - enableComplex = true; + if (rewriteTypes != null) { + switch (rewriteTypes[i]) { + case DISABLE: + enableComplex = false; + break; + case ENABLE: + enableComplex = true; } } SqlNode newOperand = operand.accept(CompoundIdentifierConverter.this); @@ -124,64 +187,4 @@ public class CompoundIdentifierConverter extends SqlShuttle { return newOperand; } } - - static final Map<Class<? extends SqlCall>, RewriteType[]> REWRITE_RULES; - - enum RewriteType { - UNCHANGED, DISABLE, ENABLE; - } - - static { - final RewriteType E =RewriteType.ENABLE; - final RewriteType D =RewriteType.DISABLE; - final RewriteType U =RewriteType.UNCHANGED; - - /* - This map stores the rules that instruct each SqlCall class which data field needs - to be rewritten if that data field is a CompoundIdentifier - - Key : Each rule corresponds to a SqlCall class; - value: It is an array of RewriteType, each being associated with a data field - in that class. - - For example, there are four data fields (query, orderList, offset, fetch) - in org.eigenbase.sql.SqlOrderBy. Since only orderList needs to be written, - RewriteType[] should be R(D, E, D, D). - */ - Map<Class<? extends SqlCall>, RewriteType[]> rules = Maps.newHashMap(); - - //SqlNodeList keywordList, - //SqlNodeList selectList, - //SqlNode fromClause, - //SqlNode whereClause, - //SqlNodeList groupBy, - //SqlNode having, - //SqlNodeList windowDecls, - //SqlNodeList orderBy, - //SqlNode offset, - //SqlNode fetch, - rules.put(SqlSelect.class, R(D, E, D, E, E, E, E, E, D, D)); - rules.put(SqlCreateTable.class, R(D, D, D, E, D, D)); - rules.put(SqlCreateView.class, R(D, E, E, D)); - rules.put(DrillSqlDescribeTable.class, R(D, D, E)); - rules.put(SqlDropView.class, R(D, D)); - rules.put(SqlShowFiles.class, R(D)); - rules.put(SqlShowSchemas.class, R(D, D)); - rules.put(SqlUseSchema.class, R(D)); - rules.put(SqlJoin.class, R(D, D, D, D, D, E)); - rules.put(SqlOrderBy.class, R(D, E, D, D)); - rules.put(SqlDropTable.class, R(D, D)); - rules.put(SqlRefreshMetadata.class, R(D)); - rules.put(SqlSetOption.class, R(D, D, D)); - rules.put(SqlCreateFunction.class, R(D)); - rules.put(SqlDropFunction.class, R(D)); - REWRITE_RULES = ImmutableMap.copyOf(rules); - } - - // Each type in the input arguments refers to - // each data field in the class - private static RewriteType[] R(RewriteType... types){ - return types; - } - } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java index 6bf3f9a..98f051e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestE2EUnnestAndLateral.java @@ -21,6 +21,7 @@ import org.apache.drill.categories.OperatorTest; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.test.ClusterFixture; import org.apache.drill.test.ClusterTest; +import org.apache.drill.test.TestBuilder; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -167,6 +168,93 @@ public class TestE2EUnnestAndLateral extends ClusterTest { } @Test + public void testUnnestWithItem() throws Exception { + String sql = "select u.item from\n" + + "cp.`lateraljoin/nested-customer.parquet` c," + + "unnest(c.orders['items']) as u(item)\n" + + "limit 1"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("item") + .baselineValues( + TestBuilder.mapOf("i_name", "paper towel", + "i_number", 2.0, + "i_supplier", "oregan")) + .go(); + } + + @Test + public void testUnnestWithFunctionCall() throws Exception { + String sql = "select u.ord.o_amount o_amount from\n" + + "cp.`lateraljoin/nested-customer.parquet` c," + + "unnest(convert_fromjson(convert_tojson(c.orders))) as u(ord)\n" + + "limit 1"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("o_amount") + .baselineValues(4.5) + .go(); + } + + @Test + public void testUnnestWithMap() throws Exception { + String sql = "select u.item from\n" + + "cp.`lateraljoin/nested-customer.parquet` c," + + "unnest(c.orders.items) as u(item)\n" + + "limit 1"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("item") + .baselineValues( + TestBuilder.mapOf("i_name", "paper towel", + "i_number", 2.0, + "i_supplier", "oregan")) + .go(); + } + + @Test + public void testMultiUnnestWithMap() throws Exception { + String sql = "select u.item from\n" + + "cp.`lateraljoin/nested-customer.parquet` c," + + "unnest(c.orders.items) as u(item)," + + "unnest(c.orders.items) as u1(item1)\n" + + "limit 1"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("item") + .baselineValues( + TestBuilder.mapOf("i_name", "paper towel", + "i_number", 2.0, + "i_supplier", "oregan")) + .go(); + } + + @Test + public void testSingleUnnestCol() throws Exception { + String sql = + "select t.orders.o_id as id " + + "from (select u.orders from\n" + + "cp.`lateraljoin/nested-customer.parquet` c," + + "unnest(c.orders) as u(orders)\n" + + "limit 1) t"; + + testBuilder() + .sqlQuery(sql) + .unOrdered() + .baselineColumns("id") + .baselineValues(1.0) + .go(); + } + + @Test public void testNestedUnnest() throws Exception { String Sql = "select * from (select customer.orders as orders from cp.`lateraljoin/nested-customer.parquet` customer ) t1," + " lateral ( select t.ord.items as items from unnest(t1.orders) t(ord) ) t2, unnest(t2.items) t3(item) ";