Fix for project column ordering is wrong.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/f4c37bfe Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/f4c37bfe Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/f4c37bfe Branch: refs/heads/diagnostics2 Commit: f4c37bfe1b213d250f7feb73ca7ccf9b88e49255 Parents: b8cef6b Author: Jacques Nadeau <[email protected]> Authored: Tue May 20 16:39:48 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Tue May 20 16:39:48 2014 -0700 ---------------------------------------------------------------------- .../exec/planner/physical/ExchangePrel.java | 1 + .../drill/exec/planner/physical/FilterPrel.java | 1 + .../exec/planner/physical/HashAggPrel.java | 1 + .../drill/exec/planner/physical/JoinPrel.java | 1 + .../planner/physical/JoinPrelRenameVisitor.java | 75 ------------------ .../drill/exec/planner/physical/LimitPrel.java | 1 + .../drill/exec/planner/physical/Prel.java | 1 + .../exec/planner/physical/PrelVisitor.java | 30 ------- .../exec/planner/physical/ProjectPrel.java | 1 + .../drill/exec/planner/physical/ScanPrel.java | 1 + .../drill/exec/planner/physical/ScreenPrel.java | 5 +- .../physical/SelectionVectorPrelVisitor.java | 66 ---------------- .../drill/exec/planner/physical/SinglePrel.java | 1 + .../drill/exec/planner/physical/SortPrel.java | 1 + .../exec/planner/physical/StreamAggPrel.java | 1 + .../drill/exec/planner/physical/WriterPrel.java | 18 ++++- .../planner/physical/explain/PrelSequencer.java | 9 +-- .../physical/visitor/BasePrelVisitor.java | 53 +++++++++++++ .../physical/visitor/FinalColumnReorderer.java | 83 ++++++++++++++++++++ .../physical/visitor/JoinPrelRenameVisitor.java | 73 +++++++++++++++++ .../planner/physical/visitor/PrelVisitor.java | 38 +++++++++ .../visitor/SelectionVectorPrelVisitor.java | 58 ++++++++++++++ .../planner/sql/handlers/DefaultSqlHandler.java | 18 ++++- .../planner/sql/handlers/ExplainHandler.java | 3 +- 24 files changed, 353 insertions(+), 187 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ExchangePrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ExchangePrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ExchangePrel.java index 21bf3b1..5a75adb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ExchangePrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ExchangePrel.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.planner.physical; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.eigenbase.rel.RelNode; import org.eigenbase.relopt.RelOptCluster; import org.eigenbase.relopt.RelTraitSet; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java index 9632911..1b13dcd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java @@ -28,6 +28,7 @@ import org.apache.drill.exec.planner.common.DrillFilterRelBase; import org.apache.drill.exec.planner.cost.DrillCostBase; import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory; import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.eigenbase.rel.RelNode; import org.eigenbase.rel.metadata.RelMetadataQuery; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java index 6377e35..b33805c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashAggPrel.java @@ -36,6 +36,7 @@ import org.apache.drill.exec.physical.config.HashAggregate; import org.apache.drill.exec.planner.cost.DrillCostBase; import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory; import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.eigenbase.rel.AggregateCall; import org.eigenbase.rel.AggregateRelBase; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java index 9e1f6fb..0db9a97 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.List; import org.apache.drill.exec.planner.common.DrillJoinRelBase; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.eigenbase.rel.InvalidRelException; import org.eigenbase.rel.JoinRelType; import org.eigenbase.rel.RelNode; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrelRenameVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrelRenameVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrelRenameVisitor.java deleted file mode 100644 index ed4b8f8..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrelRenameVisitor.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.drill.exec.planner.physical; - -import java.util.List; - -import org.eigenbase.rel.RelNode; - -import com.google.common.collect.Lists; - -public class JoinPrelRenameVisitor implements PrelVisitor<Prel, Void, RuntimeException>{ - - private static JoinPrelRenameVisitor INSTANCE = new JoinPrelRenameVisitor(); - - public static Prel insertRenameProject(Prel prel){ - return prel.accept(INSTANCE, null); - } - - @Override - public Prel visitExchange(ExchangePrel prel, Void value) throws RuntimeException { - return visitPrel(prel, value); - } - - @Override - public Prel visitPrel(Prel prel, Void value) throws RuntimeException { - List<RelNode> children = Lists.newArrayList(); - for(Prel child : prel){ - child = child.accept(this, null); - children.add(child); - } - - return (Prel) prel.copy(prel.getTraitSet(), children); - - } - - @Override - public Prel visitJoin(JoinPrel prel, Void value) throws RuntimeException { - - List<RelNode> children = Lists.newArrayList(); - - for(Prel child : prel){ - child = child.accept(this, null); - children.add(child); - } - - final int leftCount = children.get(0).getRowType().getFieldCount(); - - List<RelNode> reNamedChildren = Lists.newArrayList(); - - RelNode left = prel.getJoinInput(0, children.get(0)); - RelNode right = prel.getJoinInput(leftCount, children.get(1)); - - reNamedChildren.add(left); - reNamedChildren.add(right); - - return (Prel) prel.copy(prel.getTraitSet(), reNamedChildren); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java index 794593a..376f334 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/LimitPrel.java @@ -30,6 +30,7 @@ import org.apache.drill.exec.planner.common.DrillLimitRelBase; import org.apache.drill.exec.planner.logical.DrillImplementor; import org.apache.drill.exec.planner.logical.DrillParseContext; import org.apache.drill.exec.planner.logical.DrillRel; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.planner.torel.ConversionContext; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.eigenbase.rel.InvalidRelException; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java index a6c6b7c..1b4eb38 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.planner.common.DrillRelNode; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.eigenbase.relopt.Convention; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelVisitor.java deleted file mode 100644 index 93f95a8..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelVisitor.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.planner.physical; - - -public interface PrelVisitor<RETURN, EXTRA, EXCEP extends Throwable> { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PrelVisitor.class); - - public RETURN visitExchange(ExchangePrel prel, EXTRA value) throws EXCEP; - - public RETURN visitJoin(JoinPrel prel, EXTRA value) throws EXCEP; - - public RETURN visitPrel(Prel prel, EXTRA value) throws EXCEP; - -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java index 70dca25..55f9f32 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java @@ -28,6 +28,7 @@ import org.apache.drill.exec.physical.config.Project; import org.apache.drill.exec.physical.config.SelectionVectorRemover; import org.apache.drill.exec.planner.common.DrillProjectRelBase; import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.eigenbase.rel.ProjectRelBase; import org.eigenbase.rel.RelNode; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java index 8461e24..445ecd5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java @@ -30,6 +30,7 @@ import org.eigenbase.rel.AbstractRelNode; import org.apache.drill.exec.planner.common.DrillScanRelBase; import org.apache.drill.exec.planner.cost.DrillCostBase; import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.apache.drill.exec.store.StoragePlugin; import org.eigenbase.rel.RelNode; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java index d02ed44..37823ab 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScreenPrel.java @@ -25,6 +25,7 @@ import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.config.Screen; import org.apache.drill.exec.physical.config.SelectionVectorRemover; import org.apache.drill.exec.planner.common.DrillScreenRelBase; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.eigenbase.rel.RelNode; import org.eigenbase.relopt.RelOptCluster; @@ -40,7 +41,7 @@ public class ScreenPrel extends DrillScreenRelBase implements Prel { } @Override - public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) { + public ScreenPrel copy(RelTraitSet traitSet, List<RelNode> inputs) { return new ScreenPrel(getCluster(), traitSet, sole(inputs)); } @@ -63,7 +64,7 @@ public class ScreenPrel extends DrillScreenRelBase implements Prel { @Override public <T, X, E extends Throwable> T accept(PrelVisitor<T, X, E> logicalVisitor, X value) throws E { - return logicalVisitor.visitPrel(this, value); + return logicalVisitor.visitScreen(this, value); } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorPrelVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorPrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorPrelVisitor.java deleted file mode 100644 index 8f2ecd4..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SelectionVectorPrelVisitor.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.planner.physical; - -import java.util.List; - -import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.eigenbase.rel.RelNode; - -import com.google.common.collect.Lists; - - -public class SelectionVectorPrelVisitor implements PrelVisitor<Prel, Void, RuntimeException>{ - - private static SelectionVectorPrelVisitor INSTANCE = new SelectionVectorPrelVisitor(); - - public static Prel addSelectionRemoversWhereNecessary(Prel prel){ - return prel.accept(INSTANCE, null); - } - - @Override - public Prel visitExchange(ExchangePrel prel, Void value) throws RuntimeException { - return visitPrel(prel, value); - } - - @Override - public Prel visitJoin(JoinPrel prel, Void value) throws RuntimeException { - return visitPrel(prel, value); - } - - @Override - public Prel visitPrel(Prel prel, Void value) throws RuntimeException { - SelectionVectorMode[] encodings = prel.getSupportedEncodings(); - List<RelNode> children = Lists.newArrayList(); - for(Prel child : prel){ - child = child.accept(this, null); - children.add(convert(encodings, child)); - } - - return (Prel) prel.copy(prel.getTraitSet(), children); - } - - private Prel convert(SelectionVectorMode[] encodings, Prel prel){ - for(SelectionVectorMode m : encodings){ - if(prel.getEncoding() == m) return prel; - } - return new SelectionVectorRemoverPrel(prel); - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SinglePrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SinglePrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SinglePrel.java index e47dc7f..c23e2a1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SinglePrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SinglePrel.java @@ -20,6 +20,7 @@ package org.apache.drill.exec.planner.physical; import java.util.Collections; import java.util.Iterator; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.eigenbase.rel.RelNode; import org.eigenbase.rel.SingleRel; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java index fa5e900..464e1bb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java @@ -26,6 +26,7 @@ import org.apache.drill.exec.physical.config.SingleMergeExchange; import org.apache.drill.exec.physical.config.Sort; import org.apache.drill.exec.planner.cost.DrillCostBase; import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.eigenbase.rel.RelCollation; import org.eigenbase.rel.RelNode; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java index a95d926..b35d1bb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/StreamAggPrel.java @@ -37,6 +37,7 @@ import org.apache.drill.exec.physical.config.StreamingAggregate; import org.apache.drill.exec.planner.cost.DrillCostBase; import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory; import org.apache.drill.exec.planner.logical.DrillParseContext; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.eigenbase.rel.AggregateCall; import org.eigenbase.rel.AggregateRelBase; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java index e948125..a7f611c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/WriterPrel.java @@ -24,19 +24,31 @@ import java.util.List; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.planner.common.DrillWriterRelBase; import org.apache.drill.exec.planner.logical.CreateTableEntry; +import org.apache.drill.exec.planner.physical.visitor.PrelVisitor; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; import org.eigenbase.rel.RelNode; import org.eigenbase.relopt.RelOptCluster; import org.eigenbase.relopt.RelTraitSet; +import org.eigenbase.reltype.RelDataType; +import org.eigenbase.sql.type.SqlTypeName; + +import com.google.common.collect.ImmutableList; +import com.google.hive12.common.collect.Lists; public class WriterPrel extends DrillWriterRelBase implements Prel { + private static final List<String> FIELD_NAMES = ImmutableList.of("Fragment", "Number of records written"); + public WriterPrel(RelOptCluster cluster, RelTraitSet traits, RelNode child, CreateTableEntry createTableEntry) { super(Prel.DRILL_PHYSICAL, cluster, traits, child, createTableEntry); + List<RelDataType> fields = Lists.newArrayList(); + fields.add(cluster.getTypeFactory().createSqlType(SqlTypeName.VARCHAR, 255)); + fields.add(cluster.getTypeFactory().createSqlType(SqlTypeName.BIGINT)); + this.rowType = cluster.getTypeFactory().createStructType(fields, FIELD_NAMES); } @Override - public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) { + public WriterPrel copy(RelTraitSet traitSet, List<RelNode> inputs) { return new WriterPrel(getCluster(), traitSet, sole(inputs), getCreateTableEntry()); } @@ -49,6 +61,7 @@ public class WriterPrel extends DrillWriterRelBase implements Prel { return g; } + @Override public Iterator<Prel> iterator() { return PrelUtil.iter(getChild()); @@ -56,7 +69,7 @@ public class WriterPrel extends DrillWriterRelBase implements Prel { @Override public <T, X, E extends Throwable> T accept(PrelVisitor<T, X, E> logicalVisitor, X value) throws E { - return logicalVisitor.visitPrel(this, value); + return logicalVisitor.visitWriter(this, value); } @Override @@ -64,6 +77,7 @@ public class WriterPrel extends DrillWriterRelBase implements Prel { return SelectionVectorMode.DEFAULT; } + @Override public SelectionVectorMode getEncoding() { return SelectionVectorMode.NONE; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/explain/PrelSequencer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/explain/PrelSequencer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/explain/PrelSequencer.java index 771546a..ebff287 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/explain/PrelSequencer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/explain/PrelSequencer.java @@ -27,14 +27,14 @@ import java.util.Queue; import org.apache.drill.exec.planner.physical.ExchangePrel; import org.apache.drill.exec.planner.physical.JoinPrel; import org.apache.drill.exec.planner.physical.Prel; -import org.apache.drill.exec.planner.physical.PrelVisitor; +import org.apache.drill.exec.planner.physical.visitor.BasePrelVisitor; import org.eigenbase.rel.RelWriter; import org.eigenbase.sql.SqlExplainLevel; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -public class PrelSequencer implements PrelVisitor<Void, PrelSequencer.Frag, RuntimeException>{ +public class PrelSequencer extends BasePrelVisitor<Void, PrelSequencer.Frag, RuntimeException>{ private List<Frag> frags = Lists.newLinkedList(); @@ -235,11 +235,6 @@ public class PrelSequencer implements PrelVisitor<Void, PrelSequencer.Frag, Runt } @Override - public Void visitJoin(JoinPrel prel, Frag value) throws RuntimeException { - return visitPrel(prel, value); - } - - @Override public Void visitPrel(Prel prel, Frag value) throws RuntimeException { for(Prel children : prel){ children.accept(this, value); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java new file mode 100644 index 0000000..e3cfecc --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.physical.visitor; + +import org.apache.drill.exec.planner.physical.ExchangePrel; +import org.apache.drill.exec.planner.physical.JoinPrel; +import org.apache.drill.exec.planner.physical.Prel; +import org.apache.drill.exec.planner.physical.ScreenPrel; +import org.apache.drill.exec.planner.physical.WriterPrel; + +public class BasePrelVisitor<RETURN, EXTRA, EXCEP extends Throwable> implements PrelVisitor<RETURN, EXTRA, EXCEP> { + + @Override + public RETURN visitExchange(ExchangePrel prel, EXTRA value) throws EXCEP { + return visitPrel(prel, value); + } + + @Override + public RETURN visitJoin(JoinPrel prel, EXTRA value) throws EXCEP { + return visitPrel(prel, value); + } + + @Override + public RETURN visitScreen(ScreenPrel prel, EXTRA value) throws EXCEP { + return visitPrel(prel, value); + } + + @Override + public RETURN visitWriter(WriterPrel prel, EXTRA value) throws EXCEP { + return visitPrel(prel, value); + } + + @Override + public RETURN visitPrel(Prel prel, EXTRA value) throws EXCEP { + throw new UnsupportedOperationException(String.format("No visit method defined for prel %s in visitor %s.", prel, this.getClass().getName())); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java new file mode 100644 index 0000000..6ed3c1f --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/FinalColumnReorderer.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.physical.visitor; + +import java.util.Collections; +import java.util.List; + +import org.apache.drill.exec.planner.physical.Prel; +import org.apache.drill.exec.planner.physical.ProjectPrel; +import org.apache.drill.exec.planner.physical.ScreenPrel; +import org.apache.drill.exec.planner.physical.WriterPrel; +import org.eigenbase.rel.RelNode; +import org.eigenbase.reltype.RelDataType; +import org.eigenbase.rex.RexBuilder; +import org.eigenbase.rex.RexNode; + +import com.google.common.collect.Lists; + +public class FinalColumnReorderer extends BasePrelVisitor<Prel, Void, RuntimeException>{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FinalColumnReorderer.class); + + private static FinalColumnReorderer INSTANCE = new FinalColumnReorderer(); + + public static Prel addFinalColumnOrdering(Prel prel){ + return prel.accept(INSTANCE, null); + } + + @Override + public Prel visitScreen(ScreenPrel prel, Void value) throws RuntimeException { + Prel newChild = ((Prel) prel.getChild()).accept(this, value); + return prel.copy(prel.getTraitSet(), Collections.singletonList( (RelNode) addTrivialOrderedProjectPrel( newChild ))); + } + + private Prel addTrivialOrderedProjectPrel(Prel prel){ + RelDataType t = prel.getRowType(); + RexBuilder b = prel.getCluster().getRexBuilder(); + List<RexNode> projections = Lists.newArrayList(); + int projectCount = t.getFieldList().size(); + for(int i =0; i < projectCount; i++){ + projections.add(b.makeInputRef(prel, i)); + } + return new ProjectPrel(prel.getCluster(), prel.getTraitSet(), prel, projections, prel.getRowType()); + } + + @Override + public Prel visitWriter(WriterPrel prel, Void value) throws RuntimeException { + Prel newChild = ((Prel) prel.getChild()).accept(this, null); + return prel.copy(prel.getTraitSet(), Collections.singletonList( (RelNode) addTrivialOrderedProjectPrel( newChild ))); + } + + @Override + public Prel visitPrel(Prel prel, Void value) throws RuntimeException { + List<RelNode> children = Lists.newArrayList(); + boolean changed = false; + for(Prel p : prel){ + Prel newP = p.accept(this, null); + if(newP != p) changed = true; + children.add(newP); + } + if(changed){ + return (Prel) prel.copy(prel.getTraitSet(), children); + }else{ + return prel; + } + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java new file mode 100644 index 0000000..3d38484 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/JoinPrelRenameVisitor.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.exec.planner.physical.visitor; + +import java.util.List; + +import org.apache.drill.exec.planner.physical.ExchangePrel; +import org.apache.drill.exec.planner.physical.JoinPrel; +import org.apache.drill.exec.planner.physical.Prel; +import org.eigenbase.rel.RelNode; + +import com.google.common.collect.Lists; + +public class JoinPrelRenameVisitor extends BasePrelVisitor<Prel, Void, RuntimeException>{ + + private static JoinPrelRenameVisitor INSTANCE = new JoinPrelRenameVisitor(); + + public static Prel insertRenameProject(Prel prel){ + return prel.accept(INSTANCE, null); + } + + @Override + public Prel visitPrel(Prel prel, Void value) throws RuntimeException { + List<RelNode> children = Lists.newArrayList(); + for(Prel child : prel){ + child = child.accept(this, null); + children.add(child); + } + + return (Prel) prel.copy(prel.getTraitSet(), children); + + } + + @Override + public Prel visitJoin(JoinPrel prel, Void value) throws RuntimeException { + + List<RelNode> children = Lists.newArrayList(); + + for(Prel child : prel){ + child = child.accept(this, null); + children.add(child); + } + + final int leftCount = children.get(0).getRowType().getFieldCount(); + + List<RelNode> reNamedChildren = Lists.newArrayList(); + + RelNode left = prel.getJoinInput(0, children.get(0)); + RelNode right = prel.getJoinInput(leftCount, children.get(1)); + + reNamedChildren.add(left); + reNamedChildren.add(right); + + return (Prel) prel.copy(prel.getTraitSet(), reNamedChildren); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java new file mode 100644 index 0000000..b834464 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.physical.visitor; + +import org.apache.drill.exec.planner.physical.ExchangePrel; +import org.apache.drill.exec.planner.physical.JoinPrel; +import org.apache.drill.exec.planner.physical.Prel; +import org.apache.drill.exec.planner.physical.ScreenPrel; +import org.apache.drill.exec.planner.physical.WriterPrel; + + +public interface PrelVisitor<RETURN, EXTRA, EXCEP extends Throwable> { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PrelVisitor.class); + + public RETURN visitExchange(ExchangePrel prel, EXTRA value) throws EXCEP; + public RETURN visitScreen(ScreenPrel prel, EXTRA value) throws EXCEP; + public RETURN visitWriter(WriterPrel prel, EXTRA value) throws EXCEP; + + public RETURN visitJoin(JoinPrel prel, EXTRA value) throws EXCEP; + + public RETURN visitPrel(Prel prel, EXTRA value) throws EXCEP; + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SelectionVectorPrelVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SelectionVectorPrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SelectionVectorPrelVisitor.java new file mode 100644 index 0000000..8d0b155 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SelectionVectorPrelVisitor.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.physical.visitor; + +import java.util.List; + +import org.apache.drill.exec.planner.physical.Prel; +import org.apache.drill.exec.planner.physical.SelectionVectorRemoverPrel; +import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; +import org.eigenbase.rel.RelNode; + +import com.google.common.collect.Lists; + + +public class SelectionVectorPrelVisitor extends BasePrelVisitor<Prel, Void, RuntimeException>{ + + private static SelectionVectorPrelVisitor INSTANCE = new SelectionVectorPrelVisitor(); + + public static Prel addSelectionRemoversWhereNecessary(Prel prel){ + return prel.accept(INSTANCE, null); + } + + @Override + public Prel visitPrel(Prel prel, Void value) throws RuntimeException { + SelectionVectorMode[] encodings = prel.getSupportedEncodings(); + List<RelNode> children = Lists.newArrayList(); + for(Prel child : prel){ + child = child.accept(this, null); + children.add(convert(encodings, child)); + } + + return (Prel) prel.copy(prel.getTraitSet(), children); + } + + private Prel convert(SelectionVectorMode[] encodings, Prel prel){ + for(SelectionVectorMode m : encodings){ + if(prel.getEncoding() == m) return prel; + } + return new SelectionVectorRemoverPrel(prel); + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java index e8bd837..29ed1ec 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java @@ -38,11 +38,12 @@ import org.apache.drill.exec.planner.logical.DrillRel; import org.apache.drill.exec.planner.logical.DrillScreenRel; import org.apache.drill.exec.planner.logical.DrillStoreRel; import org.apache.drill.exec.planner.physical.DrillDistributionTrait; -import org.apache.drill.exec.planner.physical.JoinPrelRenameVisitor; import org.apache.drill.exec.planner.physical.PhysicalPlanCreator; import org.apache.drill.exec.planner.physical.Prel; -import org.apache.drill.exec.planner.physical.SelectionVectorPrelVisitor; import org.apache.drill.exec.planner.physical.explain.PrelSequencer; +import org.apache.drill.exec.planner.physical.visitor.FinalColumnReorderer; +import org.apache.drill.exec.planner.physical.visitor.JoinPrelRenameVisitor; +import org.apache.drill.exec.planner.physical.visitor.SelectionVectorPrelVisitor; import org.apache.drill.exec.planner.sql.DrillSqlWorker; import org.apache.drill.exec.util.Pointer; import org.eigenbase.rel.RelNode; @@ -80,9 +81,13 @@ public class DefaultSqlHandler extends AbstractSqlHandler { } protected void log(String name, Prel node) { - if(textPlan != null) textPlan.value = PrelSequencer.printWithIds(node, SqlExplainLevel.ALL_ATTRIBUTES); + String plan = PrelSequencer.printWithIds(node, SqlExplainLevel.ALL_ATTRIBUTES);; + if(textPlan != null){ + textPlan.value = plan; + } + if (logger.isDebugEnabled()) { - logger.debug(name + " : \n" + textPlan.value); + logger.debug(name + " : \n" + plan); } } @@ -138,6 +143,11 @@ public class DefaultSqlHandler extends AbstractSqlHandler { // In such case, we have to insert Project to rename the conflicting names. phyRelNode = JoinPrelRenameVisitor.insertRenameProject(phyRelNode); + // Since our operators work via names rather than indices, we have to make to reorder any output + // before we return data to the user as we may have accindentally shuffled things. This adds + // a trivial project to reorder columns prior to output. + phyRelNode = FinalColumnReorderer.addFinalColumnOrdering(phyRelNode); + // the last thing we do is add any required selection vector removers given the supported encodings of each // operator. This will ultimately move to a new trait but we're managing here for now to avoid introducing new // issues in planning before the next release http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f4c37bfe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java index 86ce6c5..63db153 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java @@ -32,6 +32,7 @@ import org.apache.drill.exec.planner.logical.DrillImplementor; import org.apache.drill.exec.planner.logical.DrillParseContext; import org.apache.drill.exec.planner.logical.DrillRel; import org.apache.drill.exec.planner.physical.Prel; +import org.apache.drill.exec.planner.physical.explain.PrelSequencer; import org.apache.drill.exec.planner.sql.DirectPlan; import org.eigenbase.rel.RelNode; import org.eigenbase.relopt.RelOptUtil; @@ -111,7 +112,7 @@ public class ExplainHandler extends DefaultSqlHandler{ public String json; public PhysicalExplain(RelNode node, PhysicalPlan plan, SqlExplainLevel level, QueryContext context){ - this.text = RelOptUtil.toString(node, level); + this.text = PrelSequencer.printWithIds((Prel) node, level); this.json = plan.unparse(context.getConfig().getMapper().writer()); } }
