Thanks we worked together! On Tuesday, March 22, 2016, Sudheesh Katkam <sudhe...@apache.org> wrote:
> Thanks! > > Hsuan did most of the work; I am glad this is merged :) > > On Tue, Mar 22, 2016 at 4:51 PM, Jacques Nadeau <jacq...@apache.org > <javascript:;>> wrote: > > > Awesome job on this Sudheesh. Thanks for all the hard work. Thanks also > to > > Sean for all his work on the previous patch. > > ---------- Forwarded message ---------- > > From: <sudhe...@apache.org <javascript:;>> > > Date: Mar 22, 2016 4:33 PM > > Subject: drill git commit: DRILL-3623: For limit 0 queries, optionally > use > > a shorter execution path when result column types are known > > To: <comm...@drill.apache.org <javascript:;>> > > Cc: > > > > Repository: drill > > Updated Branches: > > refs/heads/master 600ba9ee1 -> 5dbaafbe6 > > > > > > DRILL-3623: For limit 0 queries, optionally use a shorter execution path > > when result column types are known > > > > + "planner.enable_limit0_optimization" option is disabled by default > > > > + Print plan in PlanTestBase if TEST_QUERY_PRINTING_SILENT is set > > + Fix DrillTestWrapper to verify expected and actual schema > > + Correct the schema of results in TestInbuiltHiveUDFs#testXpath_Double > > > > This closes #405 > > > > > > Project: http://git-wip-us.apache.org/repos/asf/drill/repo > > Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/5dbaafbe > > Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/5dbaafbe > > Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/5dbaafbe > > > > Branch: refs/heads/master > > Commit: 5dbaafbe6651b0a284fef69d5c952d82ce506e20 > > Parents: 600ba9e > > Author: Sudheesh Katkam <skat...@maprtech.com <javascript:;>> > > Authored: Tue Mar 22 15:21:51 2016 -0700 > > Committer: Sudheesh Katkam <skat...@maprtech.com <javascript:;>> > > Committed: Tue Mar 22 16:19:01 2016 -0700 > > > > ---------------------------------------------------------------------- > > .../drill/exec/fn/hive/TestInbuiltHiveUDFs.java | 2 +- > > .../org/apache/drill/exec/ExecConstants.java | 3 + > > .../drill/exec/physical/base/ScanStats.java | 6 +- > > .../apache/drill/exec/planner/PlannerPhase.java | 2 + > > .../planner/logical/DrillDirectScanRel.java | 70 ++ > > .../exec/planner/physical/DirectScanPrule.java | 49 ++ > > .../planner/sql/handlers/DefaultSqlHandler.java | 12 + > > .../planner/sql/handlers/FindLimit0Visitor.java | 124 +++- > > .../server/options/SystemOptionManager.java | 1 + > > .../exec/store/direct/DirectGroupScan.java | 27 +- > > .../java/org/apache/drill/DrillTestWrapper.java | 25 +- > > .../java/org/apache/drill/PlanTestBase.java | 9 +- > > .../impl/limit/TestEarlyLimit0Optimization.java | 663 > +++++++++++++++++++ > > 13 files changed, 963 insertions(+), 30 deletions(-) > > ---------------------------------------------------------------------- > > > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java > > ---------------------------------------------------------------------- > > diff --git > > > > > a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java > > > > > b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java > > index a287c89..a126aaa 100644 > > --- > > > > > a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java > > +++ > > > > > b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java > > @@ -58,7 +58,7 @@ public class TestInbuiltHiveUDFs extends HiveTestBase { > > > > final TypeProtos.MajorType majorType = > > TypeProtos.MajorType.newBuilder() > > .setMinorType(TypeProtos.MinorType.FLOAT8) > > - .setMode(TypeProtos.DataMode.REQUIRED) > > + .setMode(TypeProtos.DataMode.OPTIONAL) > > .build(); > > > > final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList(); > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java > > ---------------------------------------------------------------------- > > diff --git > > a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java > > b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java > > index b8f25ad..963934d 100644 > > --- > a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java > > +++ > b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java > > @@ -202,6 +202,9 @@ public interface ExecConstants { > > String AFFINITY_FACTOR_KEY = "planner.affinity_factor"; > > OptionValidator AFFINITY_FACTOR = new > > DoubleValidator(AFFINITY_FACTOR_KEY, 1.2d); > > > > + String EARLY_LIMIT0_OPT_KEY = "planner.enable_limit0_optimization"; > > + BooleanValidator EARLY_LIMIT0_OPT = new > > BooleanValidator(EARLY_LIMIT0_OPT_KEY, false); > > + > > String ENABLE_MEMORY_ESTIMATION_KEY = > > "planner.memory.enable_memory_estimation"; > > OptionValidator ENABLE_MEMORY_ESTIMATION = new > > BooleanValidator(ENABLE_MEMORY_ESTIMATION_KEY, false); > > > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java > > ---------------------------------------------------------------------- > > diff --git > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java > > index ba36931..1886c14 100644 > > --- > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java > > +++ > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java > > @@ -17,13 +17,13 @@ > > */ > > package org.apache.drill.exec.physical.base; > > > > - > > public class ScanStats { > > - static final org.slf4j.Logger logger = > > org.slf4j.LoggerFactory.getLogger(ScanStats.class); > > - > > +// private static final org.slf4j.Logger logger = > > org.slf4j.LoggerFactory.getLogger(ScanStats.class); > > > > public static final ScanStats TRIVIAL_TABLE = new > > ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, 20, 1, 1); > > > > + public static final ScanStats ZERO_RECORD_TABLE = new > > ScanStats(GroupScanProperty.EXACT_ROW_COUNT, 0, 1, 1); > > + > > private final long recordCount; > > private final float cpuCost; > > private final float diskCost; > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java > > ---------------------------------------------------------------------- > > diff --git > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java > > index 7ab7faf..57f2984 100644 > > --- > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java > > +++ > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java > > @@ -65,6 +65,7 @@ import > > org.apache.drill.exec.planner.logical.DrillWindowRule; > > import > > org.apache.drill.exec.planner.logical.partition.ParquetPruneScanRule; > > import org.apache.drill.exec.planner.logical.partition.PruneScanRule; > > import org.apache.drill.exec.planner.physical.ConvertCountToDirectScan; > > +import org.apache.drill.exec.planner.physical.DirectScanPrule; > > import org.apache.drill.exec.planner.physical.FilterPrule; > > import org.apache.drill.exec.planner.physical.HashAggPrule; > > import org.apache.drill.exec.planner.physical.HashJoinPrule; > > @@ -391,6 +392,7 @@ public enum PlannerPhase { > > ruleList.add(LimitUnionExchangeTransposeRule.INSTANCE); > > ruleList.add(UnionAllPrule.INSTANCE); > > ruleList.add(ValuesPrule.INSTANCE); > > + ruleList.add(DirectScanPrule.INSTANCE); > > > > if (ps.isHashAggEnabled()) { > > ruleList.add(HashAggPrule.INSTANCE); > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillDirectScanRel.java > > ---------------------------------------------------------------------- > > diff --git > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillDirectScanRel.java > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillDirectScanRel.java > > new file mode 100644 > > index 0000000..013016a > > --- /dev/null > > +++ > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillDirectScanRel.java > > @@ -0,0 +1,70 @@ > > +/** > > + * Licensed to the Apache Software Foundation (ASF) under one > > + * or more contributor license agreements. See the NOTICE file > > + * distributed with this work for additional information > > + * regarding copyright ownership. The ASF licenses this file > > + * to you under the Apache License, Version 2.0 (the > > + * "License"); you may not use this file except in compliance > > + * with the License. You may obtain a copy of the License at > > + * > > + * http://www.apache.org/licenses/LICENSE-2.0 > > + * > > + * Unless required by applicable law or agreed to in writing, software > > + * distributed under the License is distributed on an "AS IS" BASIS, > > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or > > implied. > > + * See the License for the specific language governing permissions and > > + * limitations under the License. > > + */ > > +package org.apache.drill.exec.planner.logical; > > + > > +import org.apache.calcite.plan.RelOptCluster; > > +import org.apache.calcite.plan.RelTraitSet; > > +import org.apache.calcite.rel.AbstractRelNode; > > +import org.apache.calcite.rel.RelWriter; > > +import org.apache.calcite.rel.type.RelDataType; > > +import org.apache.drill.common.logical.data.LogicalOperator; > > +import org.apache.drill.exec.planner.physical.PlannerSettings; > > +import org.apache.drill.exec.planner.physical.PrelUtil; > > +import org.apache.drill.exec.store.direct.DirectGroupScan; > > + > > +/** > > + * Logical RelNode representing a {@link DirectGroupScan}. This is not > > backed by a {@link DrillTable}, > > + * unlike {@link DrillScanRel}. > > + */ > > +public class DrillDirectScanRel extends AbstractRelNode implements > > DrillRel { > > + > > + private final DirectGroupScan groupScan; > > + private final RelDataType rowType; > > + > > + public DrillDirectScanRel(RelOptCluster cluster, RelTraitSet traitSet, > > DirectGroupScan directGroupScan, > > + RelDataType rowType) { > > + super(cluster, traitSet); > > + this.groupScan = directGroupScan; > > + this.rowType = rowType; > > + } > > + > > + @Override > > + public LogicalOperator implement(DrillImplementor implementor) { > > + return null; > > + } > > + > > + @Override > > + public RelDataType deriveRowType() { > > + return this.rowType; > > + } > > + > > + @Override > > + public RelWriter explainTerms(RelWriter pw) { > > + return super.explainTerms(pw).item("directscan", > > groupScan.getDigest()); > > + } > > + > > + @Override > > + public double getRows() { > > + final PlannerSettings settings = > > PrelUtil.getPlannerSettings(getCluster()); > > + return groupScan.getScanStats(settings).getRecordCount(); > > + } > > + > > + public DirectGroupScan getGroupScan() { > > + return groupScan; > > + } > > +} > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrule.java > > ---------------------------------------------------------------------- > > diff --git > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrule.java > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrule.java > > new file mode 100644 > > index 0000000..5c2fd29 > > --- /dev/null > > +++ > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrule.java > > @@ -0,0 +1,49 @@ > > +/** > > + * Licensed to the Apache Software Foundation (ASF) under one > > + * or more contributor license agreements. See the NOTICE file > > + * distributed with this work for additional information > > + * regarding copyright ownership. The ASF licenses this file > > + * to you under the Apache License, Version 2.0 (the > > + * "License"); you may not use this file except in compliance > > + * with the License. You may obtain a copy of the License at > > + * > > + * http://www.apache.org/licenses/LICENSE-2.0 > > + * > > + * Unless required by applicable law or agreed to in writing, software > > + * distributed under the License is distributed on an "AS IS" BASIS, > > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or > > implied. > > + * See the License for the specific language governing permissions and > > + * limitations under the License. > > + */ > > +package org.apache.drill.exec.planner.physical; > > + > > +import org.apache.calcite.plan.RelOptRule; > > +import org.apache.calcite.plan.RelOptRuleCall; > > +import org.apache.calcite.plan.RelTraitSet; > > +import org.apache.drill.exec.planner.logical.DrillDirectScanRel; > > +import org.apache.drill.exec.planner.logical.RelOptHelper; > > + > > +public class DirectScanPrule extends Prule { > > + > > + public static final RelOptRule INSTANCE = new DirectScanPrule(); > > + > > + public DirectScanPrule() { > > + super(RelOptHelper.any(DrillDirectScanRel.class), > > "Prel.DirectScanPrule"); > > + } > > + > > + @Override > > + public void onMatch(RelOptRuleCall call) { > > + final DrillDirectScanRel scan = call.rel(0); > > + final RelTraitSet traits = > > scan.getTraitSet().plus(Prel.DRILL_PHYSICAL); > > + > > + final ScanPrel newScan = new ScanPrel(scan.getCluster(), traits, > > scan.getGroupScan(), scan.getRowType()) { > > + // direct scan (no execution) => no accidental column shuffling => > > no reordering > > + @Override > > + public boolean needsFinalColumnReordering() { > > + return false; > > + } > > + }; > > + > > + call.transformTo(newScan); > > + } > > +} > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java > > ---------------------------------------------------------------------- > > diff --git > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java > > index 4ca9fe4..341bae2 100644 > > --- > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java > > +++ > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java > > @@ -206,6 +206,18 @@ public class DefaultSqlHandler extends > > AbstractSqlHandler { > > * @throws RelConversionException > > */ > > protected DrillRel convertToDrel(final RelNode relNode) throws > > SqlUnsupportedException, RelConversionException { > > + if (context.getOptions().getOption(ExecConstants.EARLY_LIMIT0_OPT) > && > > + context.getPlannerSettings().isTypeInferenceEnabled() && > > + FindLimit0Visitor.containsLimit0(relNode)) { > > + // disable distributed mode > > + context.getPlannerSettings().forceSingleMode(); > > + // if the schema is known, return the schema directly > > + final DrillRel shorterPlan; > > + if ((shorterPlan = > > FindLimit0Visitor.getDirectScanRelIfFullySchemaed(relNode)) != null) { > > + return shorterPlan; > > + } > > + } > > + > > try { > > final RelNode convertedRelNode; > > > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java > > ---------------------------------------------------------------------- > > diff --git > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java > > index d2c5fa6..fa1fe07 100644 > > --- > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java > > +++ > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java > > @@ -17,6 +17,10 @@ > > */ > > package org.apache.drill.exec.planner.sql.handlers; > > > > +import com.google.common.base.Preconditions; > > +import com.google.common.collect.ImmutableSet; > > +import com.google.common.collect.Lists; > > +import org.apache.calcite.plan.RelTraitSet; > > import org.apache.calcite.rel.RelNode; > > import org.apache.calcite.rel.RelShuttleImpl; > > import org.apache.calcite.rel.logical.LogicalAggregate; > > @@ -25,10 +29,27 @@ import org.apache.calcite.rel.logical.LogicalJoin; > > import org.apache.calcite.rel.logical.LogicalMinus; > > import org.apache.calcite.rel.logical.LogicalSort; > > import org.apache.calcite.rel.logical.LogicalUnion; > > +import org.apache.calcite.rel.type.RelDataTypeField; > > import org.apache.calcite.rex.RexLiteral; > > import org.apache.calcite.rex.RexNode; > > import org.apache.calcite.sql.SqlKind; > > +import org.apache.calcite.sql.type.SqlTypeName; > > +import org.apache.drill.common.exceptions.ExecutionSetupException; > > +import org.apache.drill.common.types.TypeProtos; > > +import org.apache.drill.exec.exception.SchemaChangeException; > > +import org.apache.drill.exec.expr.TypeHelper; > > +import org.apache.drill.exec.ops.OperatorContext; > > +import org.apache.drill.exec.physical.base.ScanStats; > > +import org.apache.drill.exec.physical.impl.OutputMutator; > > +import org.apache.drill.exec.planner.logical.DrillDirectScanRel; > > import org.apache.drill.exec.planner.logical.DrillLimitRel; > > +import org.apache.drill.exec.planner.logical.DrillRel; > > +import org.apache.drill.exec.planner.sql.TypeInferenceUtils; > > +import org.apache.drill.exec.record.MaterializedField; > > +import org.apache.drill.exec.store.AbstractRecordReader; > > +import org.apache.drill.exec.store.direct.DirectGroupScan; > > + > > +import java.util.List; > > > > /** > > * Visitor that will identify whether the root portion of the RelNode > tree > > contains a limit 0 pattern. In this case, we > > @@ -36,16 +57,68 @@ import > > org.apache.drill.exec.planner.logical.DrillLimitRel; > > * executing a schema-only query. > > */ > > public class FindLimit0Visitor extends RelShuttleImpl { > > - private static final org.slf4j.Logger logger = > > org.slf4j.LoggerFactory.getLogger(FindLimit0Visitor.class); > > +// private static final org.slf4j.Logger logger = > > org.slf4j.LoggerFactory.getLogger(FindLimit0Visitor.class); > > + > > + // Some types are excluded in this set: > > + // + DECIMAL type is not fully supported in general. > > + // + VARBINARY is not fully tested. > > + // + MAP, ARRAY are currently not exposed to the planner. > > + // + TINYINT, SMALLINT are defined in the Drill type system but have > > been turned off for now. > > + // + SYMBOL, MULTISET, DISTINCT, STRUCTURED, ROW, OTHER, CURSOR, > > COLUMN_LIST are Calcite types > > + // currently not supported by Drill, nor defined in the Drill type > > list. > > + // + ANY is the late binding type. > > + private static final ImmutableSet<SqlTypeName> TYPES = > > + ImmutableSet.<SqlTypeName>builder() > > + .add(SqlTypeName.INTEGER, SqlTypeName.BIGINT, > SqlTypeName.FLOAT, > > SqlTypeName.DOUBLE, > > + SqlTypeName.VARCHAR, SqlTypeName.BOOLEAN, > SqlTypeName.DATE, > > SqlTypeName.TIME, > > + SqlTypeName.TIMESTAMP, SqlTypeName.INTERVAL_YEAR_MONTH, > > SqlTypeName.INTERVAL_DAY_TIME, > > + SqlTypeName.CHAR) > > + .build(); > > + > > + /** > > + * If all field types of the given node are {@link #TYPES recognized > > types} and honored by execution, then this > > + * method returns the tree: DrillDirectScanRel(field types). > Otherwise, > > the method returns null. > > + * > > + * @param rel calcite logical rel tree > > + * @return drill logical rel tree > > + */ > > + public static DrillRel getDirectScanRelIfFullySchemaed(RelNode rel) { > > + final List<RelDataTypeField> fieldList = > > rel.getRowType().getFieldList(); > > + final List<SqlTypeName> columnTypes = Lists.newArrayList(); > > + final List<TypeProtos.DataMode> dataModes = Lists.newArrayList(); > > + > > + for (final RelDataTypeField field : fieldList) { > > + final SqlTypeName sqlTypeName = field.getType().getSqlTypeName(); > > + if (!TYPES.contains(sqlTypeName)) { > > + return null; > > + } else { > > + columnTypes.add(sqlTypeName); > > + dataModes.add(field.getType().isNullable() ? > > + TypeProtos.DataMode.OPTIONAL : > TypeProtos.DataMode.REQUIRED); > > + } > > + } > > > > - private boolean contains = false; > > + final RelTraitSet traits = > > rel.getTraitSet().plus(DrillRel.DRILL_LOGICAL); > > + final RelDataTypeReader reader = new > > RelDataTypeReader(rel.getRowType().getFieldNames(), columnTypes, > > + dataModes); > > + return new DrillDirectScanRel(rel.getCluster(), traits, > > + new DirectGroupScan(reader, ScanStats.ZERO_RECORD_TABLE), > > rel.getRowType()); > > + } > > > > + /** > > + * Check if the root portion of the tree contains LIMIT(0). > > + * > > + * @param rel rel node tree > > + * @return true if the root portion of the tree contains LIMIT(0) > > + */ > > public static boolean containsLimit0(RelNode rel) { > > FindLimit0Visitor visitor = new FindLimit0Visitor(); > > rel.accept(visitor); > > return visitor.isContains(); > > } > > > > + private boolean contains = false; > > + > > private FindLimit0Visitor() { > > } > > > > @@ -53,7 +126,7 @@ public class FindLimit0Visitor extends RelShuttleImpl > { > > return contains; > > } > > > > - private boolean isLimit0(RexNode fetch) { > > + private static boolean isLimit0(RexNode fetch) { > > if (fetch != null && fetch.isA(SqlKind.LITERAL)) { > > RexLiteral l = (RexLiteral) fetch; > > switch (l.getTypeName()) { > > @@ -116,4 +189,49 @@ public class FindLimit0Visitor extends > RelShuttleImpl > > { > > public RelNode visit(LogicalUnion union) { > > return union; > > } > > + > > + /** > > + * Reader for column names and types. > > + */ > > + public static class RelDataTypeReader extends AbstractRecordReader { > > + > > + public final List<String> columnNames; > > + public final List<SqlTypeName> columnTypes; > > + public final List<TypeProtos.DataMode> dataModes; > > + > > + public RelDataTypeReader(List<String> columnNames, List<SqlTypeName> > > columnTypes, > > + List<TypeProtos.DataMode> dataModes) { > > + Preconditions.checkArgument(columnNames.size() == > columnTypes.size() > > && > > + columnTypes.size() == dataModes.size()); > > + this.columnNames = columnNames; > > + this.columnTypes = columnTypes; > > + this.dataModes = dataModes; > > + } > > + > > + @Override > > + public void setup(OperatorContext context, OutputMutator output) > > throws ExecutionSetupException { > > + for (int i = 0; i < columnNames.size(); i++) { > > + final TypeProtos.MajorType type = > > TypeProtos.MajorType.newBuilder() > > + .setMode(dataModes.get(i)) > > + > > > > > .setMinorType(TypeInferenceUtils.getDrillTypeFromCalciteType(columnTypes.get(i))) > > + .build(); > > + final MaterializedField field = > > MaterializedField.create(columnNames.get(i), type); > > + final Class vvClass = > > TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()); > > + try { > > + output.addField(field, vvClass); > > + } catch (SchemaChangeException e) { > > + throw new ExecutionSetupException(e); > > + } > > + } > > + } > > + > > + @Override > > + public int next() { > > + return 0; > > + } > > + > > + @Override > > + public void close() throws Exception { > > + } > > + } > > } > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java > > ---------------------------------------------------------------------- > > diff --git > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java > > index cbc5c09..a596d3a 100644 > > --- > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java > > +++ > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java > > @@ -116,6 +116,7 @@ public class SystemOptionManager extends > > BaseOptionManager implements AutoClosea > > ExecConstants.SMALL_QUEUE_SIZE, > > ExecConstants.MIN_HASH_TABLE_SIZE, > > ExecConstants.MAX_HASH_TABLE_SIZE, > > + ExecConstants.EARLY_LIMIT0_OPT, > > ExecConstants.ENABLE_MEMORY_ESTIMATION, > > ExecConstants.MAX_QUERY_MEMORY_PER_NODE, > > ExecConstants.NON_BLOCKING_OPERATORS_MEMORY, > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java > > ---------------------------------------------------------------------- > > diff --git > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java > > index e08fe71..a4b2fad 100644 > > --- > > > > > a/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java > > +++ > > > > > b/exec/java-exec/src/main/java/org/apache/drill/exec/store/direct/DirectGroupScan.java > > @@ -17,12 +17,9 @@ > > */ > > package org.apache.drill.exec.store.direct; > > > > -import java.util.Collections; > > -import java.util.List; > > - > > +import com.fasterxml.jackson.annotation.JsonTypeName; > > import org.apache.drill.common.exceptions.ExecutionSetupException; > > import org.apache.drill.common.expression.SchemaPath; > > -import org.apache.drill.exec.physical.EndpointAffinity; > > import org.apache.drill.exec.physical.PhysicalOperatorSetupException; > > import org.apache.drill.exec.physical.base.AbstractGroupScan; > > import org.apache.drill.exec.physical.base.GroupScan; > > @@ -32,14 +29,23 @@ import org.apache.drill.exec.physical.base.SubScan; > > import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; > > import org.apache.drill.exec.store.RecordReader; > > > > -public class DirectGroupScan extends AbstractGroupScan{ > > - static final org.slf4j.Logger logger = > > org.slf4j.LoggerFactory.getLogger(DirectGroupScan.class); > > +import java.util.List; > > + > > +@JsonTypeName("direct-scan") > > +public class DirectGroupScan extends AbstractGroupScan { > > +// private static final org.slf4j.Logger logger = > > org.slf4j.LoggerFactory.getLogger(DirectGroupScan.class); > > > > private final RecordReader reader; > > + private final ScanStats stats; > > > > public DirectGroupScan(RecordReader reader) { > > - super((String)null); > > + this(reader, ScanStats.TRIVIAL_TABLE); > > + } > > + > > + public DirectGroupScan(RecordReader reader, ScanStats stats) { > > + super((String) null); > > this.reader = reader; > > + this.stats = stats; > > } > > > > @Override > > @@ -58,14 +64,15 @@ public class DirectGroupScan extends > AbstractGroupScan{ > > return 1; > > } > > > > - public ScanStats getScanStats(){ > > - return ScanStats.TRIVIAL_TABLE; > > + @Override > > + public ScanStats getScanStats() { > > + return stats; > > } > > > > @Override > > public PhysicalOperator getNewWithChildren(List<PhysicalOperator> > > children) throws ExecutionSetupException { > > assert children == null || children.isEmpty(); > > - return new DirectGroupScan(reader); > > + return new DirectGroupScan(reader, stats); > > } > > > > @Override > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java > > ---------------------------------------------------------------------- > > diff --git > > a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java > > b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java > > index 67017ce..f853414 100644 > > --- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java > > +++ b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java > > @@ -31,8 +31,10 @@ import java.util.Map; > > import java.util.Set; > > import java.util.TreeMap; > > > > +import org.apache.commons.lang3.tuple.Pair; > > import org.apache.drill.common.expression.SchemaPath; > > import org.apache.drill.common.types.TypeProtos; > > +import org.apache.drill.common.types.Types; > > import org.apache.drill.exec.HyperVectorValueIterator; > > import org.apache.drill.exec.exception.SchemaChangeException; > > import org.apache.drill.exec.memory.BufferAllocator; > > @@ -294,9 +296,7 @@ public class DrillTestWrapper { > > > > protected void compareSchemaOnly() throws Exception { > > RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); > > - List<QueryDataBatch> actual = Collections.EMPTY_LIST; > > - > > - > > + List<QueryDataBatch> actual; > > QueryDataBatch batch = null; > > try { > > BaseTestQuery.test(testOptionSettingQueries); > > @@ -305,21 +305,24 @@ public class DrillTestWrapper { > > loader.load(batch.getHeader().getDef(), batch.getData()); > > > > final BatchSchema schema = loader.getSchema(); > > - if(schema.getFieldCount() != > testBuilder.getExpectedSchema().size()) > > { > > - throw new Exception("The column numbers for actual schema and > > expected schema do not match"); > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema > = > > testBuilder.getExpectedSchema(); > > + if(schema.getFieldCount() != expectedSchema.size()) { > > + throw new Exception("Expected and actual numbers of columns do > not > > match."); > > } > > > > for(int i = 0; i < schema.getFieldCount(); ++i) { > > final String actualSchemaPath = schema.getColumn(i).getPath(); > > final TypeProtos.MajorType actualMajorType = > > schema.getColumn(i).getType(); > > > > - final String expectedSchemaPath = schema.getColumn(i).getPath(); > > - final TypeProtos.MajorType expectedlMajorType = > > schema.getColumn(i).getType(); > > + final String expectedSchemaPath = > > expectedSchema.get(i).getLeft().getAsUnescapedPath(); > > + final TypeProtos.MajorType expectedMajorType = > > expectedSchema.get(i).getValue(); > > > > - if(!actualSchemaPath.equalsIgnoreCase(expectedSchemaPath) > > - || !actualMajorType.equals(expectedlMajorType)) { > > - throw new Exception("The type of the " + i + "-th column is > '" + > > actualSchemaPath + "' mismatched, expected: '" > > - + expectedlMajorType + "'"); > > + if(!actualSchemaPath.equals(expectedSchemaPath) > > + || !actualMajorType.equals(expectedMajorType)) { > > + throw new Exception(String.format("Schema path or type > mismatch > > for column #%d:\n" + > > + "Expected schema path: %s\nActual schema path: > > %s\nExpected type: %s\nActual type: %s", > > + i, expectedSchemaPath, actualSchemaPath, > > Types.toString(expectedMajorType), > > + Types.toString(actualMajorType))); > > } > > } > > > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java > > ---------------------------------------------------------------------- > > diff --git > > a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java > > b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java > > index 3922a38..bb5ff88 100644 > > --- a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java > > +++ b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java > > @@ -291,6 +291,7 @@ public class PlanTestBase extends BaseTestQuery { > > final List<QueryDataBatch> results = testSqlWithResults(sql); > > final RecordBatchLoader loader = new > > RecordBatchLoader(getDrillbitContext().getAllocator()); > > final StringBuilder builder = new StringBuilder(); > > + final boolean silent = config != null && > > config.getBoolean(QueryTestUtil.TEST_QUERY_PRINTING_SILENT); > > > > for (final QueryDataBatch b : results) { > > if (!b.hasData()) { > > @@ -308,12 +309,16 @@ public class PlanTestBase extends BaseTestQuery { > > throw new Exception("Looks like you did not provide an explain > > plan query, please add EXPLAIN PLAN FOR to the beginning of your > query."); > > } > > > > - System.out.println(vw.getValueVector().getField().getPath()); > > + if (!silent) { > > + System.out.println(vw.getValueVector().getField().getPath()); > > + } > > final ValueVector vv = vw.getValueVector(); > > for (int i = 0; i < vv.getAccessor().getValueCount(); i++) { > > final Object o = vv.getAccessor().getObject(i); > > builder.append(o); > > - System.out.println(vv.getAccessor().getObject(i)); > > + if (!silent) { > > + System.out.println(o); > > + } > > } > > loader.clear(); > > b.release(); > > > > > > > http://git-wip-us.apache.org/repos/asf/drill/blob/5dbaafbe/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java > > ---------------------------------------------------------------------- > > diff --git > > > > > a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java > > > > > b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java > > new file mode 100644 > > index 0000000..70b0cb3 > > --- /dev/null > > +++ > > > > > b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestEarlyLimit0Optimization.java > > @@ -0,0 +1,663 @@ > > +/** > > + * Licensed to the Apache Software Foundation (ASF) under one > > + * or more contributor license agreements. See the NOTICE file > > + * distributed with this work for additional information > > + * regarding copyright ownership. The ASF licenses this file > > + * to you under the Apache License, Version 2.0 (the > > + * "License"); you may not use this file except in compliance > > + * with the License. You may obtain a copy of the License at > > + * <p/> > > + * http://www.apache.org/licenses/LICENSE-2.0 > > + * <p/> > > + * Unless required by applicable law or agreed to in writing, software > > + * distributed under the License is distributed on an "AS IS" BASIS, > > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or > > implied. > > + * See the License for the specific language governing permissions and > > + * limitations under the License. > > + */ > > +package org.apache.drill.exec.physical.impl.limit; > > + > > +import com.google.common.collect.Lists; > > +import org.apache.commons.lang3.tuple.Pair; > > +import org.apache.drill.BaseTestQuery; > > +import org.apache.drill.PlanTestBase; > > +import org.apache.drill.common.expression.SchemaPath; > > +import org.apache.drill.common.types.TypeProtos; > > +import org.apache.drill.common.types.Types; > > +import org.apache.drill.exec.ExecConstants; > > +import org.joda.time.DateTime; > > +import org.junit.After; > > +import org.junit.AfterClass; > > +import org.junit.Before; > > +import org.junit.BeforeClass; > > +import org.junit.Test; > > + > > +import java.sql.Date; > > +import java.util.List; > > + > > +public class TestEarlyLimit0Optimization extends BaseTestQuery { > > + > > + private static final String viewName = "limitZeroEmployeeView"; > > + > > + private static String wrapLimit0(final String query) { > > + return "SELECT * FROM (" + query + ") LZT LIMIT 0"; > > + } > > + > > + @BeforeClass > > + public static void createView() throws Exception { > > + test("USE dfs_test.tmp"); > > + test(String.format("CREATE OR REPLACE VIEW %s AS SELECT " + > > + "CAST(employee_id AS INT) AS employee_id, " + > > + "CAST(full_name AS VARCHAR(25)) AS full_name, " + > > + "CAST(position_id AS INTEGER) AS position_id, " + > > + "CAST(department_id AS BIGINT) AS department_id," + > > + "CAST(birth_date AS DATE) AS birth_date, " + > > + "CAST(hire_date AS TIMESTAMP) AS hire_date, " + > > + "CAST(salary AS DOUBLE) AS salary, " + > > + "CAST(salary AS FLOAT) AS fsalary, " + > > + "CAST((CASE WHEN marital_status = 'S' THEN true ELSE false END) > AS > > BOOLEAN) AS single, " + > > + "CAST(education_level AS VARCHAR(60)) AS education_level," + > > + "CAST(gender AS CHAR) AS gender " + > > + "FROM cp.`employee.json` " + > > + "ORDER BY employee_id " + > > + "LIMIT 1;", viewName)); > > + // { "employee_id":1,"full_name":"Sheri > > Nowmer","first_name":"Sheri","last_name":"Nowmer","position_id":1, > > + // > > > > > "position_title":"President","store_id":0,"department_id":1,"birth_date":"1961-08-26", > > + // "hire_date":"1994-12-01 > > 00:00:00.0","end_date":null,"salary":80000.0000,"supervisor_id":0, > > + // "education_level":"Graduate > > Degree","marital_status":"S","gender":"F","management_role":"Senior > > Management" } > > + } > > + > > + @AfterClass > > + public static void tearDownView() throws Exception { > > + test("DROP VIEW " + viewName + ";"); > > + } > > + > > + @Before > > + public void setOption() throws Exception { > > + test("SET `%s` = true;", ExecConstants.EARLY_LIMIT0_OPT_KEY); > > + } > > + > > + @After > > + public void resetOption() throws Exception { > > + test("RESET `%s`;", ExecConstants.EARLY_LIMIT0_OPT_KEY); > > + } > > + > > + // -------------------- SIMPLE QUERIES -------------------- > > + > > + @Test > > + public void infoSchema() throws Exception { > > + testBuilder() > > + .sqlQuery(String.format("DESCRIBE %s", viewName)) > > + .unOrdered() > > + .baselineColumns("COLUMN_NAME", "DATA_TYPE", "IS_NULLABLE") > > + .baselineValues("employee_id", "INTEGER", "YES") > > + .baselineValues("full_name", "CHARACTER VARYING", "YES") > > + .baselineValues("position_id", "INTEGER", "YES") > > + .baselineValues("department_id", "BIGINT", "YES") > > + .baselineValues("birth_date", "DATE", "YES") > > + .baselineValues("hire_date", "TIMESTAMP", "YES") > > + .baselineValues("salary", "DOUBLE", "YES") > > + .baselineValues("fsalary", "FLOAT", "YES") > > + .baselineValues("single", "BOOLEAN", "NO") > > + .baselineValues("education_level", "CHARACTER VARYING", "YES") > > + .baselineValues("gender", "CHARACTER", "YES") > > + .go(); > > + } > > + > > + @Test > > + public void simpleSelect() throws Exception { > > + testBuilder() > > + .sqlQuery(String.format("SELECT * FROM %s", viewName)) > > + .ordered() > > + .baselineColumns("employee_id", "full_name", "position_id", > > "department_id", "birth_date", "hire_date", > > + "salary", "fsalary", "single", "education_level", "gender") > > + .baselineValues(1, "Sheri Nowmer", 1, 1L, new > > DateTime(Date.valueOf("1961-08-26").getTime()), > > + new DateTime(Date.valueOf("1994-12-01").getTime()), > 80000.0D, > > 80000.0F, true, "Graduate Degree", "F") > > + .go(); > > + } > > + > > + @Test > > + public void simpleSelectLimit0() throws Exception { > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("employee_id"), Types.optional( > > TypeProtos.MinorType.INT)), > > + Pair.of(SchemaPath.getSimplePath("full_name"), > > Types.optional(TypeProtos.MinorType.VARCHAR)), > > + Pair.of(SchemaPath.getSimplePath("position_id"), Types.optional( > > TypeProtos.MinorType.INT)), > > + Pair.of(SchemaPath.getSimplePath("department_id"), > > Types.optional(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("birth_date"), > > Types.optional(TypeProtos.MinorType.DATE)), > > + Pair.of(SchemaPath.getSimplePath("hire_date"), > > Types.optional(TypeProtos.MinorType.TIMESTAMP)), > > + Pair.of(SchemaPath.getSimplePath("salary"), > > Types.optional(TypeProtos.MinorType.FLOAT8)), > > + Pair.of(SchemaPath.getSimplePath("fsalary"), > > Types.optional(TypeProtos.MinorType.FLOAT4)), > > + Pair.of(SchemaPath.getSimplePath("single"), > > Types.required(TypeProtos.MinorType.BIT)), > > + Pair.of(SchemaPath.getSimplePath("education_level"), > > Types.optional(TypeProtos.MinorType.VARCHAR)), > > + Pair.of(SchemaPath.getSimplePath("gender"), > > Types.optional(TypeProtos.MinorType.VARCHAR))); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(String.format("SELECT * FROM %s", > viewName))) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized("SELECT * FROM " + viewName); > > + } > > + > > + private static void checkThatQueryPlanIsOptimized(final String query) > > throws Exception { > > + PlanTestBase.testPlanMatchingPatterns( > > + wrapLimit0(query), > > + new String[]{ > > + ".*Project.*\n" + > > + ".*Scan.*RelDataTypeReader.*" > > + }, > > + new String[]{}); > > + } > > + > > + // -------------------- AGGREGATE FUNC. QUERIES -------------------- > > + > > + private static String getAggQuery(final String functionName) { > > + return "SELECT " + > > + functionName + "(employee_id) AS e, " + > > + functionName + "(position_id) AS p, " + > > + functionName + "(department_id) AS d, " + > > + functionName + "(salary) AS s, " + > > + functionName + "(fsalary) AS f " + > > + "FROM " + viewName; > > + } > > + > > + @Test > > + public void sums() throws Exception { > > + final String query = getAggQuery("SUM"); > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("e"), > > Types.optional(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("p"), > > Types.optional(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("d"), > > Types.optional(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("s"), > > Types.optional(TypeProtos.MinorType.FLOAT8)), > > + Pair.of(SchemaPath.getSimplePath("f"), > > Types.optional(TypeProtos.MinorType.FLOAT8))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .ordered() > > + .baselineColumns("e", "p", "d", "s", "f") > > + .baselineValues(1L, 1L, 1L, 80000D, 80000D) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void counts() throws Exception { > > + final String query = getAggQuery("COUNT"); > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("e"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("p"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("d"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("s"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("f"), > > Types.required(TypeProtos.MinorType.BIGINT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .baselineColumns("e", "p", "d", "s", "f") > > + .ordered() > > + .baselineValues(1L, 1L, 1L, 1L, 1L) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + private void minAndMaxTest(final String functionName) throws > Exception { > > + final String query = getAggQuery(functionName); > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("e"), Types.optional( > > TypeProtos.MinorType.INT)), > > + Pair.of(SchemaPath.getSimplePath("p"), Types.optional( > > TypeProtos.MinorType.INT)), > > + Pair.of(SchemaPath.getSimplePath("d"), > > Types.optional(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("s"), > > Types.optional(TypeProtos.MinorType.FLOAT8)), > > + Pair.of(SchemaPath.getSimplePath("f"), > > Types.optional(TypeProtos.MinorType.FLOAT4))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .baselineColumns("e", "p", "d", "s", "f") > > + .ordered() > > + .baselineValues(1, 1, 1L, 80_000D, 80_000F) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void mins() throws Exception { > > + minAndMaxTest("MIN"); > > + } > > + > > + @Test > > + public void maxs() throws Exception { > > + minAndMaxTest("MAX"); > > + } > > + > > + @Test > > + public void avgs() throws Exception { > > + final String query = getAggQuery("AVG"); > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("e"), > > Types.optional(TypeProtos.MinorType.FLOAT8)), > > + Pair.of(SchemaPath.getSimplePath("p"), > > Types.optional(TypeProtos.MinorType.FLOAT8)), > > + Pair.of(SchemaPath.getSimplePath("d"), > > Types.optional(TypeProtos.MinorType.FLOAT8)), > > + Pair.of(SchemaPath.getSimplePath("s"), > > Types.optional(TypeProtos.MinorType.FLOAT8)), > > + Pair.of(SchemaPath.getSimplePath("f"), > > Types.optional(TypeProtos.MinorType.FLOAT8))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .ordered() > > + .baselineColumns("e", "p", "d", "s", "f") > > + .baselineValues(1D, 1D, 1D, 80_000D, 80_000D) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void measures() throws Exception { > > + final String query = "SELECT " + > > + "STDDEV_SAMP(employee_id) AS s, " + > > + "STDDEV_POP(position_id) AS p, " + > > + "AVG(position_id) AS a, " + > > + "COUNT(position_id) AS c " + > > + "FROM " + viewName; > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("s"), > > Types.optional(TypeProtos.MinorType.FLOAT8)), > > + Pair.of(SchemaPath.getSimplePath("p"), > > Types.optional(TypeProtos.MinorType.FLOAT8)), > > + Pair.of(SchemaPath.getSimplePath("a"), > > Types.optional(TypeProtos.MinorType.FLOAT8)), > > + Pair.of(SchemaPath.getSimplePath("c"), > > Types.required(TypeProtos.MinorType.BIGINT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .ordered() > > + .baselineColumns("s", "p", "a", "c") > > + .baselineValues(null, 0.0D, 1.0D, 1L) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void nullableCount() throws Exception { > > + final String query = "SELECT " + > > + "COUNT(CASE WHEN position_id = 1 THEN NULL ELSE position_id END) > > AS c FROM " + viewName; > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("c"), > > Types.required(TypeProtos.MinorType.BIGINT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .ordered() > > + .baselineColumns("c") > > + .baselineValues(0L) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void nullableSumAndCount() throws Exception { > > + final String query = "SELECT " + > > + "COUNT(position_id) AS c, " + > > + "SUM(CAST((CASE WHEN position_id = 1 THEN NULL ELSE position_id > > END) AS INT)) AS p " + > > + "FROM " + viewName; > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("c"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("p"), > > Types.optional(TypeProtos.MinorType.BIGINT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .ordered() > > + .baselineColumns("c", "p") > > + .baselineValues(1L, null) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void castSum() throws Exception { > > + final String query = "SELECT CAST(SUM(position_id) AS INT) AS s FROM > > cp.`employee.json`"; > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("s"), Types.optional( > > TypeProtos.MinorType.INT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .ordered() > > + .baselineColumns("s") > > + .baselineValues(18422) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void sumCast() throws Exception { > > + final String query = "SELECT SUM(CAST(position_id AS INT)) AS s FROM > > cp.`employee.json`"; > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("s"), > > Types.optional(TypeProtos.MinorType.BIGINT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .ordered() > > + .baselineColumns("s") > > + .baselineValues(18422L) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void sumsAndCounts1() throws Exception { > > + final String query = "SELECT " + > > + "COUNT(*) as cs, " + > > + "COUNT(1) as c1, " + > > + "COUNT(employee_id) as cc, " + > > + "SUM(1) as s1," + > > + "department_id " + > > + " FROM " + viewName + " GROUP BY department_id"; > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("cs"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("c1"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("cc"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("s1"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("department_id"), > > Types.optional(TypeProtos.MinorType.BIGINT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .ordered() > > + .baselineColumns("cs", "c1", "cc", "s1", "department_id") > > + .baselineValues(1L, 1L, 1L, 1L, 1L) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void sumsAndCounts2() throws Exception { > > + final String query = "SELECT " + > > + "SUM(1) as s1, " + > > + "COUNT(1) as c1, " + > > + "COUNT(*) as cs, " + > > + "COUNT(CAST(n_regionkey AS INT)) as cc " + > > + "FROM cp.`tpch/nation.parquet` " + > > + "GROUP BY CAST(n_regionkey AS INT)"; > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("s1"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("c1"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("cs"), > > Types.required(TypeProtos.MinorType.BIGINT)), > > + Pair.of(SchemaPath.getSimplePath("cc"), > > Types.required(TypeProtos.MinorType.BIGINT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .ordered() > > + .baselineColumns("s1", "c1", "cs", "cc") > > + .baselineValues(5L, 5L, 5L, 5L) > > + .baselineValues(5L, 5L, 5L, 5L) > > + .baselineValues(5L, 5L, 5L, 5L) > > + .baselineValues(5L, 5L, 5L, 5L) > > + .baselineValues(5L, 5L, 5L, 5L) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + > > + } > > + > > + @Test > > + public void rank() throws Exception { > > + final String query = "SELECT RANK() OVER(PARTITION BY employee_id > > ORDER BY employee_id) AS r FROM " + viewName; > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("r"), > > Types.required(TypeProtos.MinorType.BIGINT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .ordered() > > + .baselineColumns("r") > > + .baselineValues(1L) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + // -------------------- SCALAR FUNC. QUERIES -------------------- > > + > > + @Test > > + public void cast() throws Exception { > > + final String query = "SELECT CAST(fsalary AS DOUBLE) AS d," + > > + "CAST(employee_id AS BIGINT) AS e FROM " + viewName; > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("d"), > > Types.optional(TypeProtos.MinorType.FLOAT8)), > > + Pair.of(SchemaPath.getSimplePath("e"), > > Types.optional(TypeProtos.MinorType.BIGINT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .baselineColumns("d", "e") > > + .ordered() > > + .baselineValues(80_000D, 1L) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + public void concatTest(final String query) throws Exception { > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("c"), > > Types.optional(TypeProtos.MinorType.VARCHAR))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .baselineColumns("c") > > + .ordered() > > + .baselineValues("Sheri NowmerGraduate Degree") > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void concat() throws Exception { > > + concatTest("SELECT CONCAT(full_name, education_level) AS c FROM " + > > viewName); > > + } > > + > > + @Test > > + public void concatOp() throws Exception { > > + concatTest("SELECT full_name || education_level AS c FROM " + > > viewName); > > + } > > + > > + @Test > > + public void extract() throws Exception { > > + final String query = "SELECT EXTRACT(YEAR FROM hire_date) AS e FROM > " > > + viewName; > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("e"), > > Types.optional(TypeProtos.MinorType.BIGINT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .baselineColumns("e") > > + .ordered() > > + .baselineValues(1994L) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void binary() throws Exception { > > + final String query = "SELECT " + > > + "single AND true AS b, " + > > + "full_name || education_level AS c, " + > > + "position_id / position_id AS d, " + > > + "position_id = position_id AS e, " + > > + "position_id > position_id AS g, " + > > + "position_id >= position_id AS ge, " + > > + "position_id IN (0, 1) AS i, +" + > > + "position_id < position_id AS l, " + > > + "position_id <= position_id AS le, " + > > + "position_id - position_id AS m, " + > > + "position_id * position_id AS mu, " + > > + "position_id <> position_id AS n, " + > > + "single OR false AS o, " + > > + "position_id + position_id AS p FROM " + viewName; > > + > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("b"), > > Types.required(TypeProtos.MinorType.BIT)), > > + Pair.of(SchemaPath.getSimplePath("c"), > > Types.optional(TypeProtos.MinorType.VARCHAR)), > > + Pair.of(SchemaPath.getSimplePath("d"), Types.optional( > > TypeProtos.MinorType.INT)), > > + Pair.of(SchemaPath.getSimplePath("e"), > > Types.optional(TypeProtos.MinorType.BIT)), > > + Pair.of(SchemaPath.getSimplePath("g"), > > Types.optional(TypeProtos.MinorType.BIT)), > > + Pair.of(SchemaPath.getSimplePath("ge"), > > Types.optional(TypeProtos.MinorType.BIT)), > > + Pair.of(SchemaPath.getSimplePath("i"), > > Types.optional(TypeProtos.MinorType.BIT)), > > + Pair.of(SchemaPath.getSimplePath("l"), > > Types.optional(TypeProtos.MinorType.BIT)), > > + Pair.of(SchemaPath.getSimplePath("le"), > > Types.optional(TypeProtos.MinorType.BIT)), > > + Pair.of(SchemaPath.getSimplePath("m"), Types.optional( > > TypeProtos.MinorType.INT)), > > + Pair.of(SchemaPath.getSimplePath("mu"), Types.optional( > > TypeProtos.MinorType.INT)), > > + Pair.of(SchemaPath.getSimplePath("n"), > > Types.optional(TypeProtos.MinorType.BIT)), > > + Pair.of(SchemaPath.getSimplePath("o"), > > Types.required(TypeProtos.MinorType.BIT)), > > + Pair.of(SchemaPath.getSimplePath("p"), Types.optional( > > TypeProtos.MinorType.INT))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .baselineColumns("b", "c", "d", "e", "g", "ge", "i", "l", "le", > > "m", "mu", "n", "o", "p") > > + .ordered() > > + .baselineValues(true, "Sheri NowmerGraduate Degree", 1, true, > > false, true, true, false, true, > > + 0, 1, false, true, 2) > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + public void substringTest(final String query) throws Exception { > > + @SuppressWarnings("unchecked") > > + final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = > > Lists.newArrayList( > > + Pair.of(SchemaPath.getSimplePath("s"), > > Types.optional(TypeProtos.MinorType.VARCHAR))); > > + > > + testBuilder() > > + .sqlQuery(query) > > + .baselineColumns("s") > > + .ordered() > > + .baselineValues("Sheri") > > + .go(); > > + > > + testBuilder() > > + .sqlQuery(wrapLimit0(query)) > > + .schemaBaseLine(expectedSchema) > > + .go(); > > + > > + checkThatQueryPlanIsOptimized(query); > > + } > > + > > + @Test > > + public void substring() throws Exception { > > + substringTest("SELECT SUBSTRING(full_name, 1, 5) AS s FROM " + > > viewName); > > + } > > + > > + @Test > > + public void substr() throws Exception { > > + substringTest("SELECT SUBSTR(full_name, 1, 5) AS s FROM " + > viewName); > > + } > > +} > > >
