DRILL-6118: Handle item star columns during project / filter push down and directory pruning
1. Added DrillFilterItemStarReWriterRule to re-write item star fields to regular field references. 2. Refactored DrillPushProjectIntoScanRule to handle item star fields, factored out helper classes and methods from PreUitl.class. 3. Fixed issue with dynamic star usage (after Calcite upgrade old usage of star was still present, replaced WILDCARD -> DYNAMIC_STAR for clarity). 4. Added unit tests to check project / filter push down and directory pruning with item star. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9073aed6 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9073aed6 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9073aed6 Branch: refs/heads/master Commit: 9073aed67d89e8b2188870d6c812706085c9c41b Parents: 50efb80 Author: Arina Ielchiieva <[email protected]> Authored: Thu Dec 21 19:31:00 2017 +0200 Committer: Aman Sinha <[email protected]> Committed: Sat Feb 24 19:56:35 2018 -0800 ---------------------------------------------------------------------- .../org/apache/drill/exec/dotdrill/View.java | 5 +- .../impl/project/ProjectRecordBatch.java | 12 +- .../impl/values/ValuesBatchCreator.java | 5 +- .../apache/drill/exec/planner/PlannerPhase.java | 7 +- .../drill/exec/planner/StarColumnHelper.java | 13 +- .../exec/planner/common/DrillScanRelBase.java | 10 +- .../DrillFilterItemStarReWriterRule.java | 232 +++++++++++++++++ .../drill/exec/planner/logical/DrillOptiq.java | 3 +- .../planner/logical/DrillPushProjIntoScan.java | 97 ------- .../logical/DrillPushProjectIntoScanRule.java | 256 +++++++++++++++++++ .../planner/logical/FieldsReWriterUtil.java | 138 ++++++++++ .../planner/logical/PreProcessLogicalRel.java | 6 +- .../drill/exec/planner/physical/PrelUtil.java | 247 +----------------- .../visitor/SplitUpComplexExpressions.java | 6 +- .../planner/sql/handlers/SqlHandlerUtil.java | 4 +- .../exec/planner/types/RelDataTypeHolder.java | 13 +- .../text/compliant/RepeatedVarCharOutput.java | 7 +- .../drill/exec/store/mock/MockGroupScanPOP.java | 2 +- .../parquet2/DrillParquetGroupConverter.java | 12 +- .../exec/store/text/DrillTextRecordReader.java | 4 +- .../org/apache/drill/exec/util/Utilities.java | 39 ++- .../exec/vector/complex/fn/FieldSelection.java | 13 +- .../apache/drill/exec/TestWindowFunctions.java | 4 +- .../physical/unit/MiniPlanUnitTestBase.java | 3 +- .../physical/unit/PhysicalOpUnitTestBase.java | 3 +- .../TestPushDownAndPruningWithItemStar.java | 183 +++++++++++++ .../drill/common/expression/SchemaPath.java | 12 +- 27 files changed, 923 insertions(+), 413 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java index 3524d73..615e3bc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java @@ -23,6 +23,7 @@ import org.apache.calcite.avatica.util.TimeUnit; import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.planner.StarColumnHelper; import org.apache.drill.exec.planner.types.RelDataTypeDrillImpl; import org.apache.drill.exec.planner.types.RelDataTypeHolder; @@ -72,9 +73,9 @@ public class View { @JsonProperty("fractionalSecondPrecision") Integer fractionalSecondPrecision, @JsonProperty("isNullable") Boolean isNullable) { // Fix for views which were created on Calcite 1.4. - // After Calcite upgrade star "*" was changed on dynamic star "**" + // After Calcite upgrade star "*" was changed on dynamic star "**" (SchemaPath.DYNAMIC_STAR) // and type of star was changed to SqlTypeName.DYNAMIC_STAR - this.name = "*".equals(name) ? "**" : name; + this.name = "*".equals(name) ? SchemaPath.DYNAMIC_STAR : name; this.type = "*".equals(name) && type == SqlTypeName.ANY ? SqlTypeName.DYNAMIC_STAR : type; this.precision = precision; this.scale = scale; http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index 89e0ee9..a96dfe1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -300,7 +300,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { return false; } final NameSegment expr = ((SchemaPath)ex.getExpr()).getRootSegment(); - return expr.getPath().contains(SchemaPath.WILDCARD); + return expr.getPath().contains(SchemaPath.DYNAMIC_STAR); } private void setupNewSchemaFromInput(RecordBatch incomingBatch) throws SchemaChangeException { @@ -542,7 +542,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { final NameSegment expr = ((SchemaPath) ex.getExpr()).getRootSegment(); final NameSegment ref = ex.getRef().getRootSegment(); final boolean refHasPrefix = ref.getPath().contains(StarColumnHelper.PREFIX_DELIMITER); - final boolean exprContainsStar = expr.getPath().contains(SchemaPath.WILDCARD); + final boolean exprContainsStar = expr.getPath().contains(SchemaPath.DYNAMIC_STAR); if (refHasPrefix || exprContainsStar) { needed = true; @@ -596,10 +596,10 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { final NameSegment ref = ex.getRef().getRootSegment(); final boolean exprHasPrefix = expr.getPath().contains(StarColumnHelper.PREFIX_DELIMITER); final boolean refHasPrefix = ref.getPath().contains(StarColumnHelper.PREFIX_DELIMITER); - final boolean exprIsStar = expr.getPath().equals(SchemaPath.WILDCARD); - final boolean refContainsStar = ref.getPath().contains(SchemaPath.WILDCARD); - final boolean exprContainsStar = expr.getPath().contains(SchemaPath.WILDCARD); - final boolean refEndsWithStar = ref.getPath().endsWith(SchemaPath.WILDCARD); + final boolean exprIsStar = expr.getPath().equals(SchemaPath.DYNAMIC_STAR); + final boolean refContainsStar = ref.getPath().contains(SchemaPath.DYNAMIC_STAR); + final boolean exprContainsStar = expr.getPath().contains(SchemaPath.DYNAMIC_STAR); + final boolean refEndsWithStar = ref.getPath().endsWith(SchemaPath.DYNAMIC_STAR); String exprPrefix = EMPTY_STRING; String exprSuffix = expr.getPath(); http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/values/ValuesBatchCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/values/ValuesBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/values/ValuesBatchCreator.java index c2bcab0..6b0bb41 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/values/ValuesBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/values/ValuesBatchCreator.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -36,7 +36,8 @@ public class ValuesBatchCreator implements BatchCreator<Values> { throws ExecutionSetupException { assert children.isEmpty(); - JSONRecordReader reader = new JSONRecordReader(context, config.getContent().asNode(), null, Collections.singletonList(SchemaPath.getSimplePath("*"))); + JSONRecordReader reader = new JSONRecordReader(context, config.getContent().asNode(), + null, Collections.singletonList(SchemaPath.STAR_COLUMN)); return new ScanBatch(config, context, Collections.singletonList((RecordReader) reader)); } } http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java index 28ac51e..18dfb35 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java @@ -38,9 +38,10 @@ import org.apache.drill.exec.planner.logical.DrillMergeProjectRule; import org.apache.drill.exec.planner.logical.DrillProjectRule; import org.apache.drill.exec.planner.logical.DrillPushFilterPastProjectRule; import org.apache.drill.exec.planner.logical.DrillPushLimitToScanRule; -import org.apache.drill.exec.planner.logical.DrillPushProjIntoScan; +import org.apache.drill.exec.planner.logical.DrillPushProjectIntoScanRule; import org.apache.drill.exec.planner.logical.DrillPushProjectPastFilterRule; import org.apache.drill.exec.planner.logical.DrillPushProjectPastJoinRule; +import org.apache.drill.exec.planner.logical.DrillFilterItemStarReWriterRule; import org.apache.drill.exec.planner.logical.DrillReduceAggregatesRule; import org.apache.drill.exec.planner.logical.DrillReduceExpressionsRule; import org.apache.drill.exec.planner.logical.DrillRelFactories; @@ -276,7 +277,7 @@ public enum PlannerPhase { // Due to infinite loop in planning (DRILL-3257), temporarily disable this rule //DrillProjectSetOpTransposeRule.INSTANCE, RuleInstance.PROJECT_WINDOW_TRANSPOSE_RULE, - DrillPushProjIntoScan.INSTANCE, + DrillPushProjectIntoScanRule.INSTANCE, /* Convert from Calcite Logical to Drill Logical Rules. @@ -336,6 +337,7 @@ public enum PlannerPhase { static RuleSet getPruneScanRules(OptimizerRulesContext optimizerRulesContext) { final ImmutableSet<RelOptRule> pruneRules = ImmutableSet.<RelOptRule>builder() .add( + DrillFilterItemStarReWriterRule.INSTANCE, PruneScanRule.getDirFilterOnProject(optimizerRulesContext), PruneScanRule.getDirFilterOnScan(optimizerRulesContext), ParquetPruneScanRule.getFilterOnProjectParquet(optimizerRulesContext), @@ -376,6 +378,7 @@ public enum PlannerPhase { static RuleSet getDirPruneScanRules(OptimizerRulesContext optimizerRulesContext) { final ImmutableSet<RelOptRule> pruneRules = ImmutableSet.<RelOptRule>builder() .add( + DrillFilterItemStarReWriterRule.INSTANCE, PruneScanRule.getDirFilterOnProject(optimizerRulesContext), PruneScanRule.getDirFilterOnScan(optimizerRulesContext) ) http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java index 87cbf86..216c8d2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/StarColumnHelper.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.drill.exec.planner; import java.util.List; @@ -29,7 +28,7 @@ public class StarColumnHelper { public final static String PREFIX_DELIMITER = "\u00a6\u00a6"; - public final static String PREFIXED_STAR_COLUMN = PREFIX_DELIMITER + SchemaPath.WILDCARD; + public final static String PREFIXED_STAR_COLUMN = PREFIX_DELIMITER + SchemaPath.DYNAMIC_STAR; public static boolean containsStarColumn(RelDataType type) { if (! type.isStruct()) { @@ -38,8 +37,8 @@ public class StarColumnHelper { List<String> fieldNames = type.getFieldNames(); - for (String s : fieldNames) { - if (s.startsWith(SchemaPath.WILDCARD)) { + for (String fieldName : fieldNames) { + if (SchemaPath.DYNAMIC_STAR.equals(fieldName)) { return true; } } @@ -48,7 +47,7 @@ public class StarColumnHelper { } public static boolean containsStarColumnInProject(RelDataType inputRowType, List<RexNode> projExprs) { - if (! inputRowType.isStruct()) { + if (!inputRowType.isStruct()) { return false; } @@ -56,7 +55,7 @@ public class StarColumnHelper { if (expr instanceof RexInputRef) { String name = inputRowType.getFieldNames().get(((RexInputRef) expr).getIndex()); - if (name.startsWith(SchemaPath.WILDCARD)) { + if (SchemaPath.DYNAMIC_STAR.equals(name)) { return true; } } @@ -70,7 +69,7 @@ public class StarColumnHelper { } public static boolean isNonPrefixedStarColumn(String fieldName) { - return fieldName.startsWith(SchemaPath.WILDCARD); + return SchemaPath.DYNAMIC_STAR.equals(fieldName); } public static boolean isStarColumn(String fieldName) { http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillScanRelBase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillScanRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillScanRelBase.java index 04caed8..0331475 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillScanRelBase.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillScanRelBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,12 +18,12 @@ package org.apache.drill.exec.planner.common; import org.apache.drill.exec.planner.logical.DrillTable; -import org.apache.drill.exec.planner.logical.DrillTranslatableTable; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelTraitSet; +import org.apache.drill.exec.util.Utilities; /** * Base class for logical scan rel implemented in Drill. @@ -35,11 +35,7 @@ public abstract class DrillScanRelBase extends TableScan implements DrillRelNode public DrillScanRelBase(Convention convention, RelOptCluster cluster, RelTraitSet traits, RelOptTable table) { super(cluster, traits, table); - DrillTable unwrap = table.unwrap(DrillTable.class); - if (unwrap == null) { - unwrap = table.unwrap(DrillTranslatableTable.class).getDrillTable(); - } - this.drillTable = unwrap; + this.drillTable = Utilities.getDrillTable(table); assert drillTable != null; } http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java new file mode 100644 index 0000000..84ae674 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.logical; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.apache.calcite.adapter.enumerable.EnumerableTableScan; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.prepare.RelOptTableImpl; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexVisitorImpl; +import org.apache.calcite.schema.Table; +import org.apache.drill.exec.planner.types.RelDataTypeDrillImpl; +import org.apache.drill.exec.planner.types.RelDataTypeHolder; +import org.apache.drill.exec.util.Utilities; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.drill.exec.planner.logical.FieldsReWriterUtil.DesiredField; +import static org.apache.drill.exec.planner.logical.FieldsReWriterUtil.FieldsReWriter; + +/** + * Rule will transform filter -> project -> scan call with item star fields in filter + * into project -> filter -> project -> scan where item star fields are pushed into scan + * and replaced with actual field references. + * + * This will help partition pruning and push down rules to detect fields that can be pruned or push downed. + * Item star operator appears when sub-select or cte with star are used as source. + */ +public class DrillFilterItemStarReWriterRule extends RelOptRule { + + public static final DrillFilterItemStarReWriterRule INSTANCE = new DrillFilterItemStarReWriterRule( + RelOptHelper.some(Filter.class, RelOptHelper.some(Project.class, RelOptHelper.any( TableScan.class))), + "DrillFilterItemStarReWriterRule"); + + private DrillFilterItemStarReWriterRule(RelOptRuleOperand operand, String id) { + super(operand, id); + } + + @Override + public void onMatch(RelOptRuleCall call) { + Filter filterRel = call.rel(0); + Project projectRel = call.rel(1); + TableScan scanRel = call.rel(2); + + ItemStarFieldsVisitor itemStarFieldsVisitor = new ItemStarFieldsVisitor(filterRel.getRowType().getFieldNames()); + filterRel.getCondition().accept(itemStarFieldsVisitor); + + // there are no item fields, no need to proceed further + if (!itemStarFieldsVisitor.hasItemStarFields()) { + return; + } + + Map<String, DesiredField> itemStarFields = itemStarFieldsVisitor.getItemStarFields(); + + // create new scan + RelNode newScan = constructNewScan(scanRel, itemStarFields.keySet()); + + // combine original and new projects + List<RexNode> newProjects = new ArrayList<>(projectRel.getProjects()); + + // prepare node mapper to replace item star calls with new input field references + Map<RexNode, Integer> fieldMapper = new HashMap<>(); + + // since scan might have already some fields, new field reference index should start from the last used in scan + // NB: field reference index starts from 0 thus original field count can be taken as starting index + int index = scanRel.getRowType().getFieldCount(); + + for (DesiredField desiredField : itemStarFields.values()) { + RexInputRef inputRef = new RexInputRef(index, desiredField.getType()); + // add references to item star fields in new project + newProjects.add(inputRef); + for (RexNode node : desiredField.getNodes()) { + // if field is referenced in more then one call, add each call to field mapper + fieldMapper.put(node, index); + } + // increment index for the next node reference + index++; + } + + // create new project row type + RelDataType newProjectRowType = getNewRowType( + projectRel.getCluster().getTypeFactory(), + projectRel.getRowType().getFieldList(), + itemStarFields.keySet()); + + // create new project + RelNode newProject = new LogicalProject(projectRel.getCluster(), projectRel.getTraitSet(), newScan, newProjects, newProjectRowType); + + // transform filter condition + FieldsReWriter fieldsReWriter = new FieldsReWriter(fieldMapper); + RexNode newCondition = filterRel.getCondition().accept(fieldsReWriter); + + // create new filter + RelNode newFilter = new LogicalFilter(filterRel.getCluster(), filterRel.getTraitSet(), newProject, newCondition, ImmutableSet.<CorrelationId>of()); + + // wrap with project to have the same row type as before + Project wrapper = projectRel.copy(projectRel.getTraitSet(), newFilter, projectRel.getProjects(), projectRel.getRowType()); + + call.transformTo(wrapper); + } + + /** + * Creates new row type with merged original and new fields. + * + * @param typeFactory type factory + * @param originalFields original fields + * @param newFields new fields + * @return new row type with original and new fields + */ + private RelDataType getNewRowType(RelDataTypeFactory typeFactory, + List<RelDataTypeField> originalFields, + Collection<String> newFields) { + RelDataTypeHolder relDataTypeHolder = new RelDataTypeHolder(); + + // add original fields + for (RelDataTypeField field : originalFields) { + relDataTypeHolder.getField(typeFactory, field.getName()); + } + + // add new fields + for (String fieldName : newFields) { + relDataTypeHolder.getField(typeFactory, fieldName); + } + + return new RelDataTypeDrillImpl(relDataTypeHolder, typeFactory); + } + + /** + * Constructs new scan based on the original scan. + * Preserves all original fields and add new fields. + * + * @param scanRel original scan + * @param newFields new fields + * @return new scan with original and new fields + */ + private RelNode constructNewScan(TableScan scanRel, Collection<String> newFields) { + // create new scan row type + RelDataType newScanRowType = getNewRowType( + scanRel.getCluster().getTypeFactory(), + scanRel.getRowType().getFieldList(), + newFields); + + // create new scan + RelOptTable table = scanRel.getTable(); + Class elementType = EnumerableTableScan.deduceElementType(table.unwrap(Table.class)); + + DrillTable unwrap = Utilities.getDrillTable(table); + DrillTranslatableTable newTable = new DrillTranslatableTable( + new DynamicDrillTable(unwrap.getPlugin(), unwrap.getStorageEngineName(), unwrap.getUserName(), unwrap.getSelection())); + RelOptTableImpl newOptTableImpl = RelOptTableImpl.create(table.getRelOptSchema(), newScanRowType, newTable, ImmutableList.<String>of()); + + return new EnumerableTableScan(scanRel.getCluster(), scanRel.getTraitSet(), newOptTableImpl, elementType); + } + + /** + * Traverses given node and stores all item star fields. + * For the fields with the same name, stores original calls in a list, does not duplicate fields. + * Holds state, should not be re-used. + */ + private class ItemStarFieldsVisitor extends RexVisitorImpl<RexNode> { + + private final Map<String, DesiredField> itemStarFields = new HashMap<>(); + private final List<String> fieldNames; + + ItemStarFieldsVisitor(List<String> fieldNames) { + super(true); + this.fieldNames = fieldNames; + } + + boolean hasItemStarFields() { + return !itemStarFields.isEmpty(); + } + + Map<String, DesiredField> getItemStarFields() { + return itemStarFields; + } + + @Override + public RexNode visitCall(RexCall call) { + // need to figure out field name and index + String fieldName = FieldsReWriterUtil.getFieldNameFromItemStarField(call, fieldNames); + if (fieldName != null) { + // if there is call to the already existing field, store call, do not duplicate field + DesiredField desiredField = itemStarFields.get(fieldName); + if (desiredField == null) { + itemStarFields.put(fieldName, new DesiredField(fieldName, call.getType(), call)); + } else { + desiredField.addNode(call); + } + } + + return super.visitCall(call); + } + + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java index 7dc2050..ab073be 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java @@ -42,7 +42,6 @@ import org.apache.drill.common.types.Types; import org.apache.drill.exec.planner.StarColumnHelper; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexDynamicParam; @@ -205,7 +204,7 @@ public class DrillOptiq { // Convert expr of item[*, 'abc'] into column expression 'abc' String rootSegName = left.getRootSegment().getPath(); if (StarColumnHelper.isStarColumn(rootSegName)) { - rootSegName = rootSegName.substring(0, rootSegName.indexOf("*")); + rootSegName = rootSegName.substring(0, rootSegName.indexOf(SchemaPath.DYNAMIC_STAR)); final RexLiteral literal = (RexLiteral) call.getOperands().get(1); return SchemaPath.getSimplePath(rootSegName + literal.getValue2().toString()); } http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjIntoScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjIntoScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjIntoScan.java deleted file mode 100644 index f6bc6b7..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjIntoScan.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.drill.exec.planner.logical; - -import java.io.IOException; -import java.util.List; - -import org.apache.calcite.adapter.enumerable.EnumerableTableScan; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rel.rules.ProjectRemoveRule; -import org.apache.drill.common.exceptions.DrillRuntimeException; -import org.apache.drill.exec.planner.physical.PrelUtil; -import org.apache.drill.exec.planner.physical.PrelUtil.ProjectPushInfo; -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rex.RexNode; - -import com.google.common.collect.Lists; - -public class DrillPushProjIntoScan extends RelOptRule { - public static final RelOptRule INSTANCE = new DrillPushProjIntoScan(LogicalProject.class, EnumerableTableScan.class); - - private DrillPushProjIntoScan(Class<? extends Project> projectClass, - Class<? extends TableScan> scanClass) { - super(RelOptHelper.some(projectClass, RelOptHelper.any(scanClass)), - DrillRelFactories.LOGICAL_BUILDER, "DrillPushProjIntoScan"); - } - - @Override - public void onMatch(RelOptRuleCall call) { - final Project proj = call.rel(0); - final TableScan scan = call.rel(1); - - try { - ProjectPushInfo columnInfo = PrelUtil.getColumns(scan.getRowType(), proj.getProjects()); - - // get DrillTable, either wrapped in RelOptTable, or DrillTranslatableTable. - DrillTable table = scan.getTable().unwrap(DrillTable.class); - if (table == null) { - table = scan.getTable().unwrap(DrillTranslatableTable.class).getDrillTable(); - } - - if (columnInfo == null || columnInfo.isStarQuery() // - || !table // - .getGroupScan().canPushdownProjects(columnInfo.columns)) { - return; - } - - final DrillScanRel newScan = - new DrillScanRel(scan.getCluster(), - scan.getTraitSet().plus(DrillRel.DRILL_LOGICAL), - scan.getTable(), - columnInfo.createNewRowType(proj.getInput().getCluster().getTypeFactory()), - columnInfo.columns); - - - List<RexNode> newProjects = Lists.newArrayList(); - for (RexNode n : proj.getChildExps()) { - newProjects.add(n.accept(columnInfo.getInputRewriter())); - } - - final DrillProjectRel newProj = - new DrillProjectRel(proj.getCluster(), - proj.getTraitSet().plus(DrillRel.DRILL_LOGICAL), - newScan, - newProjects, - proj.getRowType()); - - if (ProjectRemoveRule.isTrivial(newProj)) { - call.transformTo(newScan); - } else { - call.transformTo(newProj); - } - } catch (IOException e) { - throw new DrillRuntimeException(e); - } - } - -} http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectIntoScanRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectIntoScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectIntoScanRule.java new file mode 100644 index 0000000..5db1f3c --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushProjectIntoScanRule.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.logical; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import org.apache.calcite.adapter.enumerable.EnumerableTableScan; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.rules.ProjectRemoveRule; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexVisitorImpl; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.PathSegment; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.util.Utilities; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.drill.exec.planner.logical.FieldsReWriterUtil.DesiredField; +import static org.apache.drill.exec.planner.logical.FieldsReWriterUtil.FieldsReWriter; + +/** + * When table support project push down, rule can be applied to reduce number of read columns + * thus improving scan operator performance. + */ +public class DrillPushProjectIntoScanRule extends RelOptRule { + public static final RelOptRule INSTANCE = new DrillPushProjectIntoScanRule(LogicalProject.class, EnumerableTableScan.class); + + private DrillPushProjectIntoScanRule(Class<? extends Project> projectClass, + Class<? extends TableScan> scanClass) { + super(RelOptHelper.some(projectClass, RelOptHelper.any(scanClass)), + DrillRelFactories.LOGICAL_BUILDER, "DrillPushProjectIntoScanRule"); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final Project project = call.rel(0); + final TableScan scan = call.rel(1); + + try { + + if (scan.getRowType().getFieldList().isEmpty()) { + return; + } + + ProjectPushInfo projectPushInfo = getFieldsInformation(scan.getRowType(), project.getProjects()); + if (!canPushProjectIntoScan(scan.getTable(), projectPushInfo)) { + return; + } + + final DrillScanRel newScan = + new DrillScanRel(scan.getCluster(), + scan.getTraitSet().plus(DrillRel.DRILL_LOGICAL), + scan.getTable(), + projectPushInfo.createNewRowType(project.getInput().getCluster().getTypeFactory()), + projectPushInfo.getFields()); + + List<RexNode> newProjects = new ArrayList<>(); + for (RexNode n : project.getChildExps()) { + newProjects.add(n.accept(projectPushInfo.getInputReWriter())); + } + + final DrillProjectRel newProject = + new DrillProjectRel(project.getCluster(), + project.getTraitSet().plus(DrillRel.DRILL_LOGICAL), + newScan, + newProjects, + project.getRowType()); + + if (ProjectRemoveRule.isTrivial(newProject)) { + call.transformTo(newScan); + } else { + call.transformTo(newProject); + } + } catch (IOException e) { + throw new DrillRuntimeException(e); + } + } + + /** + * Push project into scan be done only if this is not a star query and + * table supports project push down. + * + * @param table table instance + * @param projectPushInfo fields information + * @return true if push project into scan can be performed, false otherwise + */ + private boolean canPushProjectIntoScan(RelOptTable table, ProjectPushInfo projectPushInfo) throws IOException { + DrillTable drillTable = Utilities.getDrillTable(table); + return !Utilities.isStarQuery(projectPushInfo.getFields()) + && drillTable.getGroupScan().canPushdownProjects(projectPushInfo.getFields()); + } + + private ProjectPushInfo getFieldsInformation(RelDataType rowType, List<RexNode> projects) { + ProjectFieldsVisitor fieldsVisitor = new ProjectFieldsVisitor(rowType); + for (RexNode exp : projects) { + PathSegment segment = exp.accept(fieldsVisitor); + fieldsVisitor.addField(segment); + } + + return fieldsVisitor.getInfo(); + } + + /** + * Stores information about fields, their names and types. + * Is responsible for creating mapper which used in field re-writer visitor. + */ + private static class ProjectPushInfo { + private final List<SchemaPath> fields; + private final FieldsReWriter reWriter; + private final List<String> fieldNames; + private final List<RelDataType> types; + + ProjectPushInfo(List<SchemaPath> fields, Map<String, DesiredField> desiredFields) { + this.fields = fields; + this.fieldNames = new ArrayList<>(); + this.types = new ArrayList<>(); + + Map<RexNode, Integer> mapper = new HashMap<>(); + + int index = 0; + for (Map.Entry<String, DesiredField> entry : desiredFields.entrySet()) { + fieldNames.add(entry.getKey()); + DesiredField desiredField = entry.getValue(); + types.add(desiredField.getType()); + for (RexNode node : desiredField.getNodes()) { + mapper.put(node, index); + } + index++; + } + this.reWriter = new FieldsReWriter(mapper); + } + + List<SchemaPath> getFields() { + return fields; + } + + FieldsReWriter getInputReWriter() { + return reWriter; + } + + /** + * Creates new row type based on stores types and field names. + * + * @param factory factory for data type descriptors. + * @return new row type + */ + RelDataType createNewRowType(RelDataTypeFactory factory) { + return factory.createStructType(types, fieldNames); + } + } + + /** + * Visitor that finds the set of inputs that are used. + */ + private static class ProjectFieldsVisitor extends RexVisitorImpl<PathSegment> { + private final List<String> fieldNames; + private final List<RelDataTypeField> fields; + + private final Set<SchemaPath> newFields = Sets.newLinkedHashSet(); + private final Map<String, DesiredField> desiredFields = new LinkedHashMap<>(); + + ProjectFieldsVisitor(RelDataType rowType) { + super(true); + this.fieldNames = rowType.getFieldNames(); + this.fields = rowType.getFieldList(); + } + + void addField(PathSegment segment) { + if (segment != null && segment instanceof PathSegment.NameSegment) { + newFields.add(new SchemaPath((PathSegment.NameSegment) segment)); + } + } + + ProjectPushInfo getInfo() { + return new ProjectPushInfo(ImmutableList.copyOf(newFields), ImmutableMap.copyOf(desiredFields)); + } + + @Override + public PathSegment visitInputRef(RexInputRef inputRef) { + int index = inputRef.getIndex(); + String name = fieldNames.get(index); + RelDataTypeField field = fields.get(index); + addDesiredField(name, field.getType(), inputRef); + return new PathSegment.NameSegment(name); + } + + @Override + public PathSegment visitCall(RexCall call) { + String itemStarFieldName = FieldsReWriterUtil.getFieldNameFromItemStarField(call, fieldNames); + if (itemStarFieldName != null) { + addDesiredField(itemStarFieldName, call.getType(), call); + return new PathSegment.NameSegment(itemStarFieldName); + } + + if (SqlStdOperatorTable.ITEM.equals(call.getOperator())) { + PathSegment mapOrArray = call.operands.get(0).accept(this); + if (mapOrArray != null) { + if (call.operands.get(1) instanceof RexLiteral) { + return mapOrArray.cloneWithNewChild(Utilities.convertLiteral((RexLiteral) call.operands.get(1))); + } + return mapOrArray; + } + } else { + for (RexNode operand : call.operands) { + addField(operand.accept(this)); + } + } + return null; + } + + private void addDesiredField(String name, RelDataType type, RexNode originalNode) { + DesiredField desiredField = desiredFields.get(name); + if (desiredField == null) { + desiredFields.put(name, new DesiredField(name, type, originalNode)); + } else { + desiredField.addNode(originalNode); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/FieldsReWriterUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/FieldsReWriterUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/FieldsReWriterUtil.java new file mode 100644 index 0000000..5ffdac0 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/FieldsReWriterUtil.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.logical; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.drill.common.expression.SchemaPath; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class FieldsReWriterUtil { + + /** + * Checks if operator call is using item star field. + * Will return field name if true. null otherwise. + * + * @param rexCall operator call + * @param fieldNames list of field names + * @return field name, null otherwise + */ + public static String getFieldNameFromItemStarField(RexCall rexCall, List<String> fieldNames) { + if (!SqlStdOperatorTable.ITEM.equals(rexCall.getOperator())) { + return null; + } + + if (rexCall.getOperands().size() != 2) { + return null; + } + + if (!(rexCall.getOperands().get(0) instanceof RexInputRef && rexCall.getOperands().get(1) instanceof RexLiteral)) { + return null; + } + + // get parent field reference from the first operand (ITEM($0, 'col_name' -> $0) + // and check if it corresponds to the dynamic star + RexInputRef rexInputRef = (RexInputRef) rexCall.getOperands().get(0); + String parentFieldName = fieldNames.get(rexInputRef.getIndex()); + if (!SchemaPath.DYNAMIC_STAR.equals(parentFieldName)) { + return null; + } + + // get field name from the second operand (ITEM($0, 'col_name') -> col_name) + RexLiteral rexLiteral = (RexLiteral) rexCall.getOperands().get(1); + if (SqlTypeName.CHAR.equals(rexLiteral.getType().getSqlTypeName())) { + return RexLiteral.stringValue(rexLiteral); + } + + return null; + } + + /** + * Holder class to store field information (name and type) with the list of nodes this field is used in. + * Primary used to hold information about new field during field re-write process. + */ + public static class DesiredField { + private final String name; + private final RelDataType type; + private final List<RexNode> nodes = new ArrayList<>(); + + public DesiredField(String name, RelDataType type, RexNode node) { + this.name = name; + this.type = type; + addNode(node); + } + + public void addNode(RexNode originalNode) { + nodes.add(originalNode); + } + + public String getName() { + return name; + } + + public RelDataType getType() { + return type; + } + + public List<RexNode> getNodes() { + return nodes; + } + } + + /** + * Replaces original node with provided in mapper, otherwise returns original node. + */ + public static class FieldsReWriter extends RexShuttle { + + private final Map<RexNode, Integer> mapper; + + public FieldsReWriter(Map<RexNode, Integer> mapper) { + this.mapper = mapper; + } + + @Override + public RexNode visitCall(final RexCall call) { + Integer index = mapper.get(call); + if (index != null) { + return new RexInputRef(index, call.getType()); + } + return super.visitCall(call); + } + + @Override + public RexNode visitInputRef(RexInputRef inputRef) { + Integer index = mapper.get(inputRef); + if (index != null) { + return new RexInputRef(index, inputRef.getType()); + } + return super.visitInputRef(inputRef); + } + + } + + +} http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/PreProcessLogicalRel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/PreProcessLogicalRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/PreProcessLogicalRel.java index 37e4ca1..f3c6ce0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/PreProcessLogicalRel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/PreProcessLogicalRel.java @@ -201,9 +201,9 @@ public class PreProcessLogicalRel extends RelShuttleImpl { @Override public RelNode visit(LogicalUnion union) { - for(RelNode child : union.getInputs()) { - for(RelDataTypeField dataField : child.getRowType().getFieldList()) { - if(dataField.getName().contains(SchemaPath.WILDCARD)) { + for (RelNode child : union.getInputs()) { + for (RelDataTypeField dataField : child.getRowType().getFieldList()) { + if (dataField.getName().contains(SchemaPath.DYNAMIC_STAR)) { unsupportedOperatorCollector.setException(SqlUnsupportedException.ExceptionType.RELATIONAL, "Union-All over schema-less tables must specify the columns explicitly\n" + "See Apache Drill JIRA: DRILL-2414"); http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java index 9c0ee40..9d07d42 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,11 +17,7 @@ */ package org.apache.drill.exec.planner.physical; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - +import com.google.common.collect.Lists; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelOptRuleCall; @@ -31,28 +27,18 @@ import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexLiteral; -import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.FieldReference; -import org.apache.drill.common.expression.PathSegment; -import org.apache.drill.common.expression.PathSegment.ArraySegment; -import org.apache.drill.common.expression.PathSegment.NameSegment; -import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.data.Order.Ordering; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import com.carrotsearch.hppc.IntIntHashMap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; public class PrelUtil { @@ -61,7 +47,7 @@ public class PrelUtil { final List<String> childFields = rowType.getFieldNames(); - for (RelFieldCollation fc: collation.getFieldCollations() ) { + for (RelFieldCollation fc : collation.getFieldCollations()) { FieldReference fr = new FieldReference(childFields.get(fc.getFieldIndex()), ExpressionPosition.UNKNOWN, false); orderExpr.add(new Ordering(fc.getDirection(), fr, fc.nullDirection)); } @@ -108,121 +94,6 @@ public class PrelUtil { return lastUsed.getLastUsedReference(); } - public static ProjectPushInfo getColumns(RelDataType rowType, List<RexNode> projects) { - final List<String> fieldNames = rowType.getFieldNames(); - if (fieldNames.isEmpty()) { - return null; - } - - RefFieldsVisitor v = new RefFieldsVisitor(rowType); - for (RexNode exp : projects) { - PathSegment segment = exp.accept(v); - v.addColumn(segment); - } - - return v.getInfo(); - - } - - public static class DesiredField { - public final int origIndex; - public final String name; - public final RelDataTypeField field; - - public DesiredField(int origIndex, String name, RelDataTypeField field) { - super(); - this.origIndex = origIndex; - this.name = name; - this.field = field; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((field == null) ? 0 : field.hashCode()); - result = prime * result + ((name == null) ? 0 : name.hashCode()); - result = prime * result + origIndex; - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - DesiredField other = (DesiredField) obj; - if (field == null) { - if (other.field != null) { - return false; - } - } else if (!field.equals(other.field)) { - return false; - } - if (name == null) { - if (other.name != null) { - return false; - } - } else if (!name.equals(other.name)) { - return false; - } - if (origIndex != other.origIndex) { - return false; - } - return true; - } - - } - - public static class ProjectPushInfo { - public final List<SchemaPath> columns; - public final List<DesiredField> desiredFields; - public final InputRewriter rewriter; - private final List<String> fieldNames; - private final List<RelDataType> types; - - public ProjectPushInfo(List<SchemaPath> columns, ImmutableList<DesiredField> desiredFields) { - super(); - this.columns = columns; - this.desiredFields = desiredFields; - - this.fieldNames = Lists.newArrayListWithCapacity(desiredFields.size()); - this.types = Lists.newArrayListWithCapacity(desiredFields.size()); - IntIntHashMap oldToNewIds = new IntIntHashMap(); - - int i =0; - for (DesiredField f : desiredFields) { - fieldNames.add(f.name); - types.add(f.field.getType()); - oldToNewIds.put(f.origIndex, i); - i++; - } - this.rewriter = new InputRewriter(oldToNewIds); - } - - public InputRewriter getInputRewriter() { - return rewriter; - } - - public boolean isStarQuery() { - for (SchemaPath column : columns) { - if (column.getRootSegment().getPath().startsWith("*")) { - return true; - } - } - return false; - } - - public RelDataType createNewRowType(RelDataTypeFactory factory) { - return factory.createStructType(types, fieldNames); - } - } // Simple visitor class to determine the last used reference in the expression private static class LastUsedRefVisitor extends RexVisitorImpl<Void> { @@ -252,69 +123,6 @@ public class PrelUtil { } } - /** Visitor that finds the set of inputs that are used. */ - private static class RefFieldsVisitor extends RexVisitorImpl<PathSegment> { - final Set<SchemaPath> columns = Sets.newLinkedHashSet(); - final private List<String> fieldNames; - final private List<RelDataTypeField> fields; - final private Set<DesiredField> desiredFields = Sets.newLinkedHashSet(); - - public RefFieldsVisitor(RelDataType rowType) { - super(true); - this.fieldNames = rowType.getFieldNames(); - this.fields = rowType.getFieldList(); - } - - public void addColumn(PathSegment segment) { - if (segment != null && segment instanceof NameSegment) { - columns.add(new SchemaPath((NameSegment)segment)); - } - } - - public ProjectPushInfo getInfo() { - return new ProjectPushInfo(ImmutableList.copyOf(columns), ImmutableList.copyOf(desiredFields)); - } - - @Override - public PathSegment visitInputRef(RexInputRef inputRef) { - int index = inputRef.getIndex(); - String name = fieldNames.get(index); - RelDataTypeField field = fields.get(index); - DesiredField f = new DesiredField(index, name, field); - desiredFields.add(f); - return new NameSegment(name); - } - - @Override - public PathSegment visitCall(RexCall call) { - if ("ITEM".equals(call.getOperator().getName())) { - PathSegment mapOrArray = call.operands.get(0).accept(this); - if (mapOrArray != null) { - if (call.operands.get(1) instanceof RexLiteral) { - return mapOrArray.cloneWithNewChild(convertLiteral((RexLiteral) call.operands.get(1))); - } - return mapOrArray; - } - } else { - for (RexNode operand : call.operands) { - addColumn(operand.accept(this)); - } - } - return null; - } - - private PathSegment convertLiteral(RexLiteral literal) { - switch (literal.getType().getSqlTypeName()) { - case CHAR: - return new NameSegment(RexLiteral.stringValue(literal)); - case INTEGER: - return new ArraySegment(RexLiteral.intValue(literal)); - default: - return null; - } - } - - } public static RelTraitSet fixTraits(RelOptRuleCall call, RelTraitSet set) { return fixTraits(call.getPlanner(), set); @@ -329,8 +137,7 @@ public class PrelUtil { } // DRILL-6089 make sure no collations are added to HashJoin - public static RelTraitSet removeCollation(RelTraitSet traitSet, RelOptRuleCall call) - { + public static RelTraitSet removeCollation(RelTraitSet traitSet, RelOptRuleCall call) { RelTraitSet newTraitSet = call.getPlanner().emptyTraitSet(); for (RelTrait trait: traitSet) { @@ -341,44 +148,4 @@ public class PrelUtil { return newTraitSet; } - - public static class InputRefRemap { - private int oldIndex; - private int newIndex; - - public InputRefRemap(int oldIndex, int newIndex) { - super(); - this.oldIndex = oldIndex; - this.newIndex = newIndex; - } - public int getOldIndex() { - return oldIndex; - } - public int getNewIndex() { - return newIndex; - } - - } - - public static class InputRewriter extends RexShuttle { - - final IntIntHashMap map; - - public InputRewriter(IntIntHashMap map) { - super(); - this.map = map; - } - - @Override - public RexNode visitInputRef(RexInputRef inputRef) { - return new RexInputRef(map.get(inputRef.getIndex()), inputRef.getType()); - } - - @Override - public RexNode visitLocalRef(RexLocalRef localRef) { - return new RexInputRef(map.get(localRef.getIndex()), localRef.getType()); - } - - } - } http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java index f323991..32241f5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -14,7 +14,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ +*/ package org.apache.drill.exec.planner.physical.visitor; import java.util.ArrayList; @@ -107,7 +107,7 @@ public class SplitUpComplexExpressions extends BasePrelVisitor<Prel, Object, Rel RexBuilder builder = new RexBuilder(factory); allExprs.add(builder.makeInputRef( new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory), index)); - if(fieldNames.get(index).contains(SchemaPath.WILDCARD)) { + if (fieldNames.get(index).contains(SchemaPath.DYNAMIC_STAR)) { relDataTypes.add(new RelDataTypeFieldImpl(fieldNames.get(index), allExprs.size(), factory.createSqlType(SqlTypeName.ANY))); } else { relDataTypes.add(new RelDataTypeFieldImpl("EXPR$" + exprIndex, allExprs.size(), factory.createSqlType(SqlTypeName.ANY))); http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java index c2227c4..134c28f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java @@ -89,7 +89,7 @@ public class SqlHandlerUtil { // CTAS's query field list shouldn't have "**" when table's field list is specified. for (String field : validatedRowtype.getFieldNames()) { - if (field.equals("**")) { + if (SchemaPath.DYNAMIC_STAR.equals(field)) { final String tblType = isNewTableView ? "view" : "table"; throw UserException.validationError() .message("%s's query field list has a '*', which is invalid when %s's field list is specified.", @@ -157,7 +157,7 @@ public class SqlHandlerUtil { .message("Partition column %s is not in the SELECT list of CTAS!", col) .build(logger); } else { - if (field.getName().startsWith(SchemaPath.WILDCARD)) { + if (SchemaPath.DYNAMIC_STAR.equals(field.getName())) { colRefStarNames.add(col); final List<RexNode> operands = Lists.newArrayList(); http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeHolder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeHolder.java index aa3542c..ad2091c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeHolder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -26,13 +26,12 @@ import org.apache.calcite.rel.type.RelDataTypeFieldImpl; import org.apache.calcite.sql.type.SqlTypeName; import com.google.common.collect.Lists; -import org.apache.calcite.util.Pair; -import org.apache.calcite.util.Util; +import org.apache.drill.common.expression.SchemaPath; public class RelDataTypeHolder { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RelDataTypeHolder.class); - List<RelDataTypeField> fields = Lists.newArrayList(); + private final List<RelDataTypeField> fields = Lists.newArrayList(); private RelDataTypeFactory typeFactory; @@ -46,9 +45,9 @@ public class RelDataTypeHolder { return fields.size(); } - private void addStarIfEmpty(RelDataTypeFactory typeFactory){ - if (fields.isEmpty()){ - getField(typeFactory, "*"); + private void addStarIfEmpty(RelDataTypeFactory typeFactory) { + if (fields.isEmpty()) { + getField(typeFactory, SchemaPath.DYNAMIC_STAR); } } http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java index 156d6c2..79321f5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,7 +27,6 @@ import java.util.Collections; import java.util.List; import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.FieldReference; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; @@ -128,13 +127,13 @@ class RepeatedVarCharOutput extends TextOutput { { // setup fields - List<Integer> columnIds = new ArrayList<Integer>(); + List<Integer> columnIds = new ArrayList<>(); if (!isStarQuery) { String pathStr; for (SchemaPath path : columns) { assert path.getRootSegment().isNamed() : "root segment should be named"; pathStr = path.getRootSegment().getPath(); - Preconditions.checkArgument(pathStr.equals(COL_NAME) || (pathStr.equals("*") && path.getRootSegment().getChild() == null), + Preconditions.checkArgument(COL_NAME.equals(pathStr) || (SchemaPath.DYNAMIC_STAR.equals(pathStr) && path.getRootSegment().getChild() == null), String.format("Selected column '%s' must have name 'columns' or must be plain '*'", pathStr)); if (path.getRootSegment().getChild() != null) { http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java index c8082a8..88c0be9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java @@ -198,7 +198,7 @@ public class MockGroupScanPOP extends AbstractGroupScan { Pattern p = Pattern.compile("(\\w+)_([isdb])(\\d*)"); for (SchemaPath path : columns) { String col = path.getLastSegment().getNameSegment().getPath(); - if (col.equals("*")) { + if (SchemaPath.DYNAMIC_STAR.equals(col)) { return this; } Matcher m = p.matcher(col); http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java index 7f50d2d..4363a7a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -117,15 +117,15 @@ public class DrillParquetGroupConverter extends GroupConverter { // Match the name of the field in the schema definition to the name of the field in the query. String name = null; - SchemaPath col=null; - PathSegment colPath = null; + SchemaPath col; + PathSegment colPath; PathSegment colNextChild = null; - while(colIterator.hasNext()) { + while (colIterator.hasNext()) { col = colIterator.next(); colPath = col.getRootSegment(); colNextChild = colPath.getChild(); - if (colPath != null && colPath.isNamed() && (!colPath.getNameSegment().getPath().equals("*"))) { + if (colPath != null && colPath.isNamed() && (!SchemaPath.DYNAMIC_STAR.equals(colPath.getNameSegment().getPath()))) { name = colPath.getNameSegment().getPath(); // We may have a field that does not exist in the schema if (!name.equalsIgnoreCase(type.getName())) { @@ -139,7 +139,7 @@ public class DrillParquetGroupConverter extends GroupConverter { } if (!isPrimitive) { - Collection<SchemaPath> c = new ArrayList<SchemaPath>(); + Collection<SchemaPath> c = new ArrayList<>(); while(colNextChild!=null) { if(colNextChild.isNamed()) { http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java index 2af5c8b..18245eb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -79,7 +79,7 @@ public class DrillTextRecordReader extends AbstractRecordReader { for (SchemaPath path : columns) { assert path.getRootSegment().isNamed(); pathStr = path.getRootSegment().getPath(); - Preconditions.checkArgument(pathStr.equals(COL_NAME) || (pathStr.equals("*") && path.getRootSegment().getChild() == null), + Preconditions.checkArgument(COL_NAME.equals(pathStr) || (SchemaPath.DYNAMIC_STAR.equals(pathStr) && path.getRootSegment().getChild() == null), "Selected column(s) must have name 'columns' or must be plain '*'"); if (path.getRootSegment().getChild() != null) { http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/util/Utilities.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/Utilities.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/Utilities.java index 9125e96..a165d9e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/Utilities.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/Utilities.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,9 +20,14 @@ package org.apache.drill.exec.util; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rex.RexLiteral; +import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.expr.fn.impl.DateUtility; import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DrillTranslatableTable; import org.apache.drill.exec.proto.BitControl.QueryContextInformation; import org.apache.drill.exec.proto.ExecProtos; import org.apache.drill.exec.proto.helper.QueryIdHelper; @@ -90,4 +95,36 @@ public class Utilities { } }).isPresent(); } + + /** + * Gets {@link DrillTable}, either wrapped in RelOptTable, or DrillTranslatableTable. + * + * @param table table instance + * @return Drill table + */ + public static DrillTable getDrillTable(RelOptTable table) { + DrillTable drillTable = table.unwrap(DrillTable.class); + if (drillTable == null) { + drillTable = table.unwrap(DrillTranslatableTable.class).getDrillTable(); + } + return drillTable; + } + + /** + * Converts literal into path segment based on its type. + * For unsupported types, returns null. + * + * @param literal literal + * @return new path segment, null otherwise + */ + public static PathSegment convertLiteral(RexLiteral literal) { + switch (literal.getType().getSqlTypeName()) { + case CHAR: + return new PathSegment.NameSegment(RexLiteral.stringValue(literal)); + case INTEGER: + return new PathSegment.ArraySegment(RexLiteral.intValue(literal)); + default: + return null; + } + } } http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/FieldSelection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/FieldSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/FieldSelection.java index acf5552..f9fad5b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/FieldSelection.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/FieldSelection.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,7 +14,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - ******************************************************************************/ + */ package org.apache.drill.exec.vector.complex.fn; import java.util.HashMap; @@ -50,10 +49,10 @@ public class FieldSelection { private FieldSelection(Map<String, FieldSelection> children, ValidityMode mode){ this.children = children; - if(children != null){ - childrenInsensitive = new TreeMap<String, FieldSelection>(String.CASE_INSENSITIVE_ORDER); + if (children != null) { + childrenInsensitive = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); childrenInsensitive.putAll(children); - }else{ + } else { childrenInsensitive = null; } this.mode = mode; @@ -148,7 +147,7 @@ public class FieldSelection { private static boolean containsStar(List<SchemaPath> columns) { for (SchemaPath expr : columns) { - if (expr.getRootSegment().getPath().equals("*")) { + if (SchemaPath.DYNAMIC_STAR.equals(expr.getRootSegment().getPath())) { return true; } } http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java index db72ec2..e2aec04 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java @@ -939,10 +939,10 @@ public class TestWindowFunctions extends BaseTestQuery { public void testWindowOnSubqueryWithStar() throws Exception { String query = "SELECT SUM(n_nationkey) OVER w as s\n" + "FROM (SELECT * FROM cp.`tpch/nation.parquet`) subQry\n" + - "WINDOW w AS (PARTITION BY REGION ORDER BY n_nationkey)\n" + + "WINDOW w AS (PARTITION BY region ORDER BY n_nationkey)\n" + "limit 1"; - final String[] expectedPlan = {"Project.*\\$0=\\[ITEM\\(\\$1, 'n_nationkey'\\)\\].*"}; + final String[] expectedPlan = {"Scan.*columns=\\[`n_nationkey`, `region`\\].*"}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, new String[]{}); testBuilder() http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java index c1f2ed5..b81d9b8 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java @@ -33,7 +33,6 @@ import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.drill.exec.store.parquet.ParquetDirectByteBufferAllocator; import org.apache.drill.exec.store.parquet.ParquetReaderUtility; import org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader; -import org.apache.drill.test.OperatorFixture; import org.apache.hadoop.fs.Path; import org.apache.parquet.hadoop.CodecFactory; import org.apache.parquet.hadoop.ParquetFileReader; @@ -318,7 +317,7 @@ public class MiniPlanUnitTestBase extends PhysicalOpUnitTestBase { } public abstract class ScanPopBuider<T extends ScanPopBuider<?>> extends PopBuilder { - List<SchemaPath> columnsToRead = Collections.singletonList(SchemaPath.getSimplePath("*")); + List<SchemaPath> columnsToRead = Collections.singletonList(SchemaPath.STAR_COLUMN); DrillFileSystem fs = null; public ScanPopBuider() { http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/PhysicalOpUnitTestBase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/PhysicalOpUnitTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/PhysicalOpUnitTestBase.java index 8f644fa..b5dbcf8 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/PhysicalOpUnitTestBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/PhysicalOpUnitTestBase.java @@ -38,7 +38,6 @@ import org.apache.drill.exec.rpc.control.Controller; import org.apache.drill.exec.rpc.control.WorkEventBus; import org.apache.drill.exec.rpc.user.UserServer; import org.apache.drill.exec.server.QueryProfileStoreContext; -import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.drill.exec.store.easy.json.JSONRecordReader; import org.apache.drill.exec.work.batch.IncomingBuffers; @@ -477,7 +476,7 @@ public class PhysicalOpUnitTestBase extends ExecTest { } private Iterator<RecordReader> getRecordReadersForJsonBatches(List<String> jsonBatches, FragmentContext fragContext) { - return getJsonReadersFromBatchString(jsonBatches, fragContext, Collections.singletonList(SchemaPath.getSimplePath("*"))); + return getJsonReadersFromBatchString(jsonBatches, fragContext, Collections.singletonList(SchemaPath.STAR_COLUMN)); } public List<RecordReader> getReaderListForJsonBatches(List<String> jsonBatches, FragmentContext fragContext) { http://git-wip-us.apache.org/repos/asf/drill/blob/9073aed6/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java new file mode 100644 index 0000000..24b9212 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.parquet; + +import org.apache.drill.PlanTestBase; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA; + +public class TestPushDownAndPruningWithItemStar extends PlanTestBase { + + private static final String TABLE_NAME = "order_ctas"; + + @BeforeClass + public static void setup() throws Exception { + test("create table `%s`.`%s/t1` as select o_orderkey, o_custkey, cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` " + + "where o_orderdate between date '1992-01-01' and date '1992-01-03'", DFS_TMP_SCHEMA, TABLE_NAME); + test("create table `%s`.`%s/t2` as select o_orderkey, o_custkey, cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` " + + "where o_orderdate between date '1992-01-04' and date '1992-01-06'", DFS_TMP_SCHEMA, TABLE_NAME); + test("create table `%s`.`%s/t3` as select o_orderkey, o_custkey, cast(o_orderdate as date) as o_orderdate from cp.`tpch/orders.parquet` " + + "where o_orderdate between date '1992-01-07' and date '1992-01-09'", DFS_TMP_SCHEMA, TABLE_NAME); + } + + @AfterClass + public static void tearDown() throws Exception { + test("drop table if exists `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME); + } + + @Test + public void testPushProjectIntoScan() throws Exception { + String query = String.format("select o_orderdate, count(*) from (select * from `%s`.`%s`) group by o_orderdate", DFS_TMP_SCHEMA, TABLE_NAME); + + String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`o_orderdate`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select o_orderdate, count(*) from `%s`.`%s` group by o_orderdate", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + @Test + public void testPushProjectIntoScanWithExpressionInProject() throws Exception { + String query = String.format("select o_custkey + o_orderkey from (select * from `%s`.`%s`)", DFS_TMP_SCHEMA, TABLE_NAME); + + String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`o_custkey`, `o_orderkey`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select o_custkey + o_orderkey from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + @Test + public void testPushProjectIntoScanWithExpressionInFilter() throws Exception { + String query = String.format("select o_orderdate from (select * from `%s`.`%s`) where o_custkey + o_orderkey < 5", DFS_TMP_SCHEMA, TABLE_NAME); + + String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`o_custkey`, `o_orderkey`, `o_orderdate`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select o_orderdate from `%s`.`%s` where o_custkey + o_orderkey < 5", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + @Test + public void testPushProjectIntoScanWithComplexInProject() throws Exception { + String query = "select t.user_info.cust_id, t.user_info.device, t.marketing_info.camp_id, t.marketing_info.keywords[2] " + + "from (select * from cp.`store/parquet/complex/complex.parquet`) t"; + + String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, " + + "columns=\\[`user_info`.`cust_id`, `user_info`.`device`, `marketing_info`.`camp_id`, `marketing_info`.`keywords`\\[2\\]\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select t.user_info.cust_id, t.user_info.device, t.marketing_info.camp_id, t.marketing_info.keywords[2] " + + "from cp.`store/parquet/complex/complex.parquet`", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + @Test + public void testPushProjectIntoScanWithComplexInFilter() throws Exception { + String query = "select t.trans_id from (select * from cp.`store/parquet/complex/complex.parquet`) t " + + "where t.user_info.cust_id > 28 and t.user_info.device = 'IOS5' and t.marketing_info.camp_id > 5 and t.marketing_info.keywords[2] is not null"; + + String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, " + + "columns=\\[`user_info`.`cust_id`, `user_info`.`device`, `marketing_info`.`camp_id`, `marketing_info`.`keywords`\\[2\\], `trans_id`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select t.trans_id from cp.`store/parquet/complex/complex.parquet` t " + + "where t.user_info.cust_id > 28 and t.user_info.device = 'IOS5' and t.marketing_info.camp_id > 5 and t.marketing_info.keywords[2] is not null", + DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + @Test + public void testDirectoryPruning() throws Exception { + String query = String.format("select * from (select * from `%s`.`%s`) where dir0 = 't1'", DFS_TMP_SCHEMA, TABLE_NAME); + + String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select * from `%s`.`%s` where dir0 = 't1'", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + @Test + public void testFilterPushDownSingleCondition() throws Exception { + String query = String.format("select * from (select * from `%s`.`%s`) where o_orderdate = date '1992-01-01'", DFS_TMP_SCHEMA, TABLE_NAME); + + String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select * from `%s`.`%s` where o_orderdate = date '1992-01-01'", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + @Test + public void testFilterPushDownMultipleConditions() throws Exception { + String query = String.format("select * from (select * from `%s`.`%s`) where o_orderdate = date '1992-01-01' or o_orderdate = date '1992-01-09'", + DFS_TMP_SCHEMA, TABLE_NAME); + + String[] expectedPlan = {"numFiles=2, numRowGroups=2, usedMetadataFile=false, columns=\\[`\\*\\*`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select * from `%s`.`%s` where o_orderdate = date '1992-01-01' or o_orderdate = date '1992-01-09'", + DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + +}
