[
https://issues.apache.org/jira/browse/DRILL-3623?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16497353#comment-16497353
]
ASF GitHub Bot commented on DRILL-3623:
---------------------------------------
ilooner closed pull request #364: DRILL-3623: Optimize limit 0 queries
URL: https://github.com/apache/drill/pull/364
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index 55de2d7363..74234ac590 100644
---
a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -19,6 +19,7 @@
import com.google.common.collect.ImmutableMap;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.PlanTestBase;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.hadoop.fs.FileSystem;
import org.joda.time.DateTime;
@@ -65,6 +66,25 @@ public void hiveReadWithDb() throws Exception {
test("select * from hive.kv");
}
+ @Test
+ public void simpleLimitZero() throws Exception {
+ testBuilder()
+ .sqlQuery("select * from hive.kv limit 0")
+ .expectsEmptyResultSet()
+ .baselineColumns("key", "value")
+ .go();
+ }
+
+ @Test
+ public void simpleLimitZeroPlan() throws Exception {
+ final String[] expectedPlan = {
+ ".*Limit.*\n" +
+ ".*Values.*"
+ };
+ final String[] excludedPlan = {};
+ PlanTestBase.testPlanMatchingPatterns("select * from hive.kv limit 0",
expectedPlan, excludedPlan);
+ }
+
@Test
public void queryEmptyHiveTable() throws Exception {
testBuilder()
@@ -188,6 +208,66 @@ public void readAllSupportedHiveDataTypes() throws
Exception {
.go();
}
+ @Test
+ public void limitZeroVariousTypes() throws Exception {
+ testBuilder().sqlQuery("SELECT * FROM (SELECT boolean_field,
tinyint_field, decimal0_field," +
+ "decimal9_field, decimal18_field, decimal28_field, decimal38_field,
double_field, float_field, int_field, " +
+ "bigint_field, smallint_field, string_field, varchar_field,
timestamp_field, date_field, " +
+ "boolean_part, tinyint_part, decimal0_part, decimal9_part,
decimal18_part, decimal28_part, decimal38_part, " +
+ "double_part, float_part, int_part, bigint_part, smallint_part,
string_part, varchar_part, timestamp_part, " +
+ "date_part FROM hive.readtest) T LIMIT 0")
+ .baselineColumns(
+ "boolean_field",
+ "tinyint_field",
+ "decimal0_field",
+ "decimal9_field",
+ "decimal18_field",
+ "decimal28_field",
+ "decimal38_field",
+ "double_field",
+ "float_field",
+ "int_field",
+ "bigint_field",
+ "smallint_field",
+ "string_field",
+ "varchar_field",
+ "timestamp_field",
+ "date_field",
+ "boolean_part",
+ "tinyint_part",
+ "decimal0_part",
+ "decimal9_part",
+ "decimal18_part",
+ "decimal28_part",
+ "decimal38_part",
+ "double_part",
+ "float_part",
+ "int_part",
+ "bigint_part",
+ "smallint_part",
+ "string_part",
+ "varchar_part",
+ "timestamp_part",
+ "date_part")
+ .expectsEmptyResultSet()
+ .go();
+ }
+
+ @Test
+ public void limitZeroVariousTypesPlan() throws Exception {
+ PlanTestBase.testPlanMatchingPatterns("SELECT * FROM (SELECT
boolean_field, tinyint_field, decimal0_field," +
+ "decimal9_field, decimal18_field, decimal28_field,
decimal38_field, double_field, float_field, int_field, " +
+ "bigint_field, smallint_field, string_field, varchar_field,
timestamp_field, date_field, " +
+ "boolean_part, tinyint_part, decimal0_part, decimal9_part,
decimal18_part, decimal28_part, decimal38_part, " +
+ "double_part, float_part, int_part, bigint_part, smallint_part,
string_part, varchar_part, timestamp_part, " +
+ "date_part FROM hive.readtest) T LIMIT 0",
+ new String[] {
+ ".*Limit.*\n" +
+ ".*Values.*"
+ },
+ new String[] {});
+ }
+
/**
* Test to ensure Drill reads the all supported types through native Parquet
readers.
* NOTE: As part of Hive 1.2 upgrade, make sure this test and {@link
#readAllSupportedHiveDataTypes()} are merged
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionRegistry.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionRegistry.java
index 05439b36ec..cbd6f2109d 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionRegistry.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionRegistry.java
@@ -108,7 +108,7 @@ public void register(DrillOperatorTable operatorTable) {
for (DrillFuncHolder func : function.getValue()) {
if (argCounts.add(func.getParamCount())) {
if (func.isAggregating()) {
- op = new DrillSqlAggOperator(name, func.getParamCount());
+ op = new DrillSqlAggOperator(name, func.getParamCount(),
func.getReturnType());
} else {
boolean isDeterministic;
// prevent Drill from folding constant functions with types that
cannot be materialized
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
index 78d27018c1..ea818af64f 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
@@ -58,6 +58,7 @@
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.NlsString;
import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.planner.sql.handlers.FindLimit0Visitor;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
@@ -70,6 +71,9 @@
private final PlannerSettings plannerSettings;
+ /**
+ * NOTE: Ensure changes are reflected in {@link FindLimit0Visitor#TYPES}.
+ */
public static ImmutableMap<TypeProtos.MinorType, SqlTypeName>
DRILL_TO_CALCITE_TYPE_MAPPING =
ImmutableMap.<TypeProtos.MinorType, SqlTypeName> builder()
.put(TypeProtos.MinorType.INT, SqlTypeName.INTEGER)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java
index a6c4661673..e9f813147e 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java
@@ -65,7 +65,7 @@
private final JSONOptions options;
private final double rowCount;
- protected DrillValuesRel(RelOptCluster cluster, RelDataType rowType,
ImmutableList<ImmutableList<RexLiteral>> tuples, RelTraitSet traits) {
+ public DrillValuesRel(RelOptCluster cluster, RelDataType rowType,
ImmutableList<ImmutableList<RexLiteral>> tuples, RelTraitSet traits) {
super(cluster, traits);
assert getConvention() == DRILL_LOGICAL;
verifyRowType(tuples, rowType);
@@ -181,7 +181,7 @@ private static void writeLiteral(RexLiteral literal,
JsonOutput out) throws IOEx
case CHAR:
if (isLiteralNull(literal)) {
- out.writeVarcharNull();
+ out.writeVarChar("");
}else{
// Since Calcite treats string literals as fixed char and adds
trailing spaces to the strings to make them the
// same length, here we do an rtrim() to get the string without the
trailing spaces. If we don't rtrim, the comparison
@@ -235,7 +235,7 @@ private static void writeLiteral(RexLiteral literal,
JsonOutput out) throws IOEx
case SYMBOL:
if (isLiteralNull(literal)) {
- out.writeVarcharNull();
+ out.writeVarChar("");
}else{
out.writeVarChar(literal.getValue().toString());
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlAggOperator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlAggOperator.java
index 213620162d..acc32b5ecd 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlAggOperator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlAggOperator.java
@@ -17,8 +17,6 @@
*/
package org.apache.drill.exec.planner.sql;
-import java.util.List;
-
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.sql.SqlAggFunction;
@@ -31,24 +29,53 @@
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;
-import com.google.common.collect.ImmutableList;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.planner.logical.DrillConstExecutor;
public class DrillSqlAggOperator extends SqlAggFunction {
- static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(DrillSqlAggOperator.class);
+ private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(DrillSqlAggOperator.class);
+ private final MajorType returnType;
- public DrillSqlAggOperator(String name, int argCount) {
+ public DrillSqlAggOperator(String name, int argCount, MajorType returnType) {
super(name, new SqlIdentifier(name, SqlParserPos.ZERO),
SqlKind.OTHER_FUNCTION, DynamicReturnType.INSTANCE, null, new
Checker(argCount), SqlFunctionCategory.USER_DEFINED_FUNCTION);
+ this.returnType = returnType;
}
- @Override
- public RelDataType deriveType(SqlValidator validator, SqlValidatorScope
scope, SqlCall call) {
- return getAny(validator.getTypeFactory());
+ private RelDataType getReturnType(final RelDataTypeFactory factory) {
+ // least restrictive type (nullable ANY type)
+ final RelDataType anyType = factory.createSqlType(SqlTypeName.ANY);
+ final RelDataType nullableAnyType =
factory.createTypeWithNullability(anyType, true);
+
+ final MinorType minorType = returnType.getMinorType();
+ final SqlTypeName sqlTypeName =
DrillConstExecutor.DRILL_TO_CALCITE_TYPE_MAPPING.get(minorType);
+ if (sqlTypeName == null) {
+ return factory.createTypeWithNullability(nullableAnyType, true);
+ }
+
+ final RelDataType relReturnType = factory.createSqlType(sqlTypeName);
+ switch (returnType.getMode()) {
+ case OPTIONAL:
+ return factory.createTypeWithNullability(relReturnType, true);
+ case REQUIRED:
+ return relReturnType;
+ case REPEATED:
+ return relReturnType;
+ default:
+ return nullableAnyType;
+ }
}
- private RelDataType getAny(RelDataTypeFactory factory){
- return factory.createSqlType(SqlTypeName.ANY);
-// return new RelDataTypeDrillImpl(new RelDataTypeHolder(), factory);
+ @Override
+ public RelDataType deriveType(SqlValidator validator, SqlValidatorScope
scope, SqlCall call) {
+ /*
+ * We return a nullable output type both in validation phase and in
+ * Sql to Rel phase. We don't know the type of the output until runtime
+ * hence have to choose the least restrictive type to avoid any wrong
+ * results.
+ */
+ return getReturnType(validator.getTypeFactory());
}
// @Override
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlOperator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlOperator.java
index 7b5a99dbc2..73c5a7f70f 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlOperator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlOperator.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p/>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -32,6 +32,7 @@
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;
+import org.apache.drill.exec.planner.logical.DrillConstExecutor;
public class DrillSqlOperator extends SqlFunction {
static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(DrillSqlOperator.class);
@@ -45,7 +46,8 @@ public DrillSqlOperator(String name, int argCount, boolean
isDeterministic) {
}
public DrillSqlOperator(String name, int argCount, MajorType returnType,
boolean isDeterminisitic) {
- super(new SqlIdentifier(name, SqlParserPos.ZERO),
DynamicReturnType.INSTANCE, null, new Checker(argCount), null,
SqlFunctionCategory.USER_DEFINED_FUNCTION);
+ super(new SqlIdentifier(name, SqlParserPos.ZERO),
DynamicReturnType.INSTANCE, null, new Checker(argCount), null,
+ SqlFunctionCategory.USER_DEFINED_FUNCTION);
this.returnType = Preconditions.checkNotNull(returnType);
this.isDeterministic = isDeterminisitic;
}
@@ -54,33 +56,47 @@ public boolean isDeterministic() {
return isDeterministic;
}
- protected RelDataType getReturnDataType(final RelDataTypeFactory factory) {
- if (MinorType.BIT.equals(returnType.getMinorType())) {
- return factory.createSqlType(SqlTypeName.BOOLEAN);
+ private RelDataType getReturnType(final RelDataTypeFactory factory) {
+ // least restrictive type (nullable ANY type)
+ final RelDataType anyType = factory.createSqlType(SqlTypeName.ANY);
+ final RelDataType nullableAnyType =
factory.createTypeWithNullability(anyType, true);
+
+ if (NONE.equals(returnType)) {
+ return nullableAnyType;
+ }
+
+ final MinorType minorType = returnType.getMinorType();
+ final SqlTypeName sqlTypeName =
DrillConstExecutor.DRILL_TO_CALCITE_TYPE_MAPPING.get(minorType);
+ if (sqlTypeName == null) {
+ return factory.createTypeWithNullability(nullableAnyType, true);
}
- return
factory.createTypeWithNullability(factory.createSqlType(SqlTypeName.ANY), true);
- }
- private RelDataType getNullableReturnDataType(final RelDataTypeFactory
factory) {
- return factory.createTypeWithNullability(getReturnDataType(factory), true);
+ final RelDataType relReturnType = factory.createSqlType(sqlTypeName);
+ switch (returnType.getMode()) {
+ case OPTIONAL:
+ return factory.createTypeWithNullability(relReturnType, true);
+ case REQUIRED:
+ return relReturnType;
+ case REPEATED:
+ return relReturnType;
+ default:
+ return nullableAnyType;
+ }
}
@Override
public RelDataType deriveType(SqlValidator validator, SqlValidatorScope
scope, SqlCall call) {
- if (NONE.equals(returnType)) {
- return validator.getTypeFactory().createSqlType(SqlTypeName.ANY);
- }
/*
* We return a nullable output type both in validation phase and in
* Sql to Rel phase. We don't know the type of the output until runtime
* hence have to choose the least restrictive type to avoid any wrong
* results.
*/
- return getNullableReturnDataType(validator.getTypeFactory());
+ return getReturnType(validator.getTypeFactory());
}
@Override
public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
- return getNullableReturnDataType(opBinding.getTypeFactory());
+ return getReturnType(opBinding.getTypeFactory());
}
}
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index d6bdc78c1c..c71b96e6b9 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -212,6 +212,15 @@ protected ConvertedRelNode validateAndConvert(SqlNode
sqlNode) throws ForemanSet
* @throws RelConversionException
*/
protected DrillRel convertToDrel(RelNode relNode) throws
SqlUnsupportedException, RelConversionException {
+ // If the query contains a limit 0 clause, disable distributed mode since
it is overkill for determining schema
+ // And if the schema is known, return the schema directly
+ if (FindLimit0Visitor.containsLimit0(relNode)) {
+ context.getPlannerSettings().forceSingleMode();
+ final DrillRel valuesRel =
FindLimit0Visitor.getValuesRelIfFullySchemaed(relNode);
+ if (valuesRel != null) {
+ return valuesRel;
+ }
+ }
try {
final DrillRel convertedRelNode;
@@ -220,12 +229,6 @@ protected DrillRel convertToDrel(RelNode relNode) throws
SqlUnsupportedException
if (convertedRelNode instanceof DrillStoreRel) {
throw new UnsupportedOperationException();
} else {
-
- // If the query contains a limit 0 clause, disable distributed mode
since it is overkill for determining schema.
- if (FindLimit0Visitor.containsLimit0(convertedRelNode)) {
- context.getPlannerSettings().forceSingleMode();
- }
-
return convertedRelNode;
}
} catch (RelOptPlanner.CannotPlanException ex) {
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
index d2c5fa68d1..38044f0595 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/FindLimit0Visitor.java
@@ -17,6 +17,9 @@
*/
package org.apache.drill.exec.planner.sql.handlers;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelShuttleImpl;
import org.apache.calcite.rel.logical.LogicalAggregate;
@@ -25,10 +28,20 @@
import org.apache.calcite.rel.logical.LogicalMinus;
import org.apache.calcite.rel.logical.LogicalSort;
import org.apache.calcite.rel.logical.LogicalUnion;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.drill.exec.planner.logical.DrillConstExecutor;
import org.apache.drill.exec.planner.logical.DrillLimitRel;
+import org.apache.drill.exec.planner.logical.DrillRel;
+import org.apache.drill.exec.planner.logical.DrillValuesRel;
+
+import java.math.BigDecimal;
+import java.util.List;
/**
* Visitor that will identify whether the root portion of the RelNode tree
contains a limit 0 pattern. In this case, we
@@ -36,16 +49,71 @@
* executing a schema-only query.
*/
public class FindLimit0Visitor extends RelShuttleImpl {
- private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(FindLimit0Visitor.class);
-
+// private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(FindLimit0Visitor.class);
+
+ /**
+ * Values in the {@link DrillConstExecutor#DRILL_TO_CALCITE_TYPE_MAPPING}
map.
+ * + without {@link SqlTypeName#ANY} (avoid late binding)
+ * + without {@link SqlTypeName#VARBINARY} ({@link DrillValuesRel values}
does not support this)
+ * + with {@link SqlTypeName#CHAR} ({@link DrillValuesRel values} supports
this, but the above mapping does not
+ * contain this type.
+ */
+ private static final ImmutableSet<SqlTypeName> TYPES =
ImmutableSet.<SqlTypeName>builder()
+ .add(SqlTypeName.INTEGER, SqlTypeName.BIGINT, /*SqlTypeName.FLOAT,*/
SqlTypeName.DOUBLE, SqlTypeName.VARCHAR,
+ SqlTypeName.BOOLEAN, SqlTypeName.DATE, /*SqlTypeName.DECIMAL,
*/SqlTypeName.TIME, SqlTypeName.TIMESTAMP,
+ SqlTypeName.INTERVAL_YEAR_MONTH, SqlTypeName.INTERVAL_DAY_TIME,
SqlTypeName.MAP, SqlTypeName.ARRAY,
+ SqlTypeName.TINYINT, SqlTypeName.SMALLINT, SqlTypeName.CHAR)
+ .build();
+
+ private static final ImmutableSet<SqlTypeName> EXTENDED_TYPES =
ImmutableSet.<SqlTypeName>builder()
+ .add(SqlTypeName.INTEGER, SqlTypeName.DATE, SqlTypeName.DECIMAL,
SqlTypeName.TIME, SqlTypeName.TIMESTAMP,
+ SqlTypeName.INTERVAL_YEAR_MONTH,
SqlTypeName.INTERVAL_DAY_TIME)
+ .build();
private boolean contains = false;
+ /**
+ * Checks if the root portion of the RelNode tree contains a limit 0 pattern.
+ */
public static boolean containsLimit0(RelNode rel) {
FindLimit0Visitor visitor = new FindLimit0Visitor();
rel.accept(visitor);
return visitor.isContains();
}
+ /**
+ * If all field types of the given node are {@link #TYPES recognized types},
then this method returns the tree:
+ * DrillLimitRel(0)
+ * \
+ * DrillValuesRel(field types)
+ * Otherwise, the method returns null.
+ */
+ public static DrillRel getValuesRelIfFullySchemaed(final RelNode rel) {
+ final List<RelDataTypeField> fieldList = rel.getRowType().getFieldList();
+ final ImmutableList.Builder<RexLiteral> tupleBuilder = new
ImmutableList.Builder<>();
+ final RexBuilder literalBuilder = new
RexBuilder(rel.getCluster().getTypeFactory());
+ for (final RelDataTypeField field : fieldList) {
+ if (!TYPES.contains(field.getType().getSqlTypeName())) {
+ return null;
+ } else {
+ RelDataType t =
rel.getCluster().getTypeFactory().createTypeWithNullability(field.getType(),
false);
+ if (!EXTENDED_TYPES.contains(field.getType().getSqlTypeName())) {
+ tupleBuilder.add((RexLiteral) literalBuilder.makeZeroLiteral(t));
+ } else {
+ tupleBuilder.add((RexLiteral) literalBuilder.makeLiteral(null,
field.getType(), false));
+ }
+ }
+ }
+
+ final ImmutableList<ImmutableList<RexLiteral>> tuples = new
ImmutableList.Builder<ImmutableList<RexLiteral>>()
+ .add(tupleBuilder.build())
+ .build();
+ final RelTraitSet traits = rel.getTraitSet().plus(DrillRel.DRILL_LOGICAL);
+ // TODO: ideally, we want the limit to be pushed into values
+ final DrillValuesRel values = new DrillValuesRel(rel.getCluster(),
rel.getRowType(), tuples, traits);
+ return new DrillLimitRel(rel.getCluster(), traits, values,
literalBuilder.makeExactLiteral(BigDecimal.ZERO),
+ literalBuilder.makeExactLiteral(BigDecimal.ZERO));
+ }
+
private FindLimit0Visitor() {
}
@@ -78,18 +146,6 @@ public RelNode visit(LogicalSort sort) {
return super.visit(sort);
}
- @Override
- public RelNode visit(RelNode other) {
- if (other instanceof DrillLimitRel) {
- if (isLimit0(((DrillLimitRel) other).getFetch())) {
- contains = true;
- return other;
- }
- }
-
- return super.visit(other);
- }
-
// The following set of RelNodes should terminate a search for the limit 0
pattern as they want convey its meaning.
@Override
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimit0.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimit0.java
new file mode 100644
index 0000000000..c15a591fdf
--- /dev/null
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestLimit0.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.limit;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.PlanTestBase;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.drill.TestBuilder.mapOf;
+
+public class TestLimit0 extends BaseTestQuery {
+
+ @Test
+ public void simpleLimit0() throws Exception {
+ List<MajorType> expectedTypes = ImmutableList.of(
+ Types.optional(MinorType.VARCHAR),
+ Types.optional(MinorType.BIGINT),
+ Types.optional(MinorType.DATE),
+ Types.optional(MinorType.TIME),
+ Types.optional(MinorType.TIMESTAMP),
+ Types.optional(MinorType.INTERVAL), // I think these are
incorrect, losing DAY/YEAR
+ Types.optional(MinorType.INTERVAL));
+
+
+ List<QueryDataBatch> results = testSqlWithResults("SELECT * FROM (" +
+ "SELECT * FROM (VALUES ('varchar', 1, DATE '2008-2-23', TIME
'12:23:34', TIMESTAMP '2008-2-23 12:23:34.456', " +
+ "INTERVAL '1' YEAR, INTERVAL '2' DAY), ('varchar', 1, DATE
'2008-2-23', TIME '12:23:34', " +
+ "TIMESTAMP '2008-2-23 12:23:34.456', INTERVAL '1' YEAR,
INTERVAL '2' DAY)) AS " +
+ "Example(myVarChar, myInt, myDate, myTime, myTimestamp, int1,
int2)) T LIMIT 0");
+ Assert.assertEquals(0, results.get(0).getHeader().getRowCount());
+ List<SerializedField> fields =
results.get(0).getHeader().getDef().getFieldList();
+ Assert.assertEquals(expectedTypes.size(), fields.size());
+ for (int i = 0; i < fields.size(); i++) {
+ Assert.assertEquals(expectedTypes.get(i), fields.get(i).getMajorType());
+ }
+ }
+
+ @Test
+ public void simpleLimit0Plan() throws Exception {
+ PlanTestBase.testPlanMatchingPatterns("SELECT * FROM (" +
+ "SELECT * FROM (VALUES (1, 1.0, DATE '2008-2-23', TIME '12:23:34',
TIMESTAMP '2008-2-23 12:23:34.456', " +
+ "INTERVAL '1' YEAR, INTERVAL '2' DAY), (1, 1.0, DATE '2008-2-23', TIME
'12:23:34', " +
+ "TIMESTAMP '2008-2-23 12:23:34.456', INTERVAL '1' YEAR, INTERVAL '2'
DAY)) AS " +
+ "Example(myInt, myFloat, myDate, myTime, myTimestamp, int1, int2)) T
LIMIT 0",
+ new String[] {
+ ".*Limit.*\n" +
+ ".*Values.*"
+ },
+ new String[] {});
+ }
+
+ @Test
+ public void countDistinctLimit0() throws Exception {
+ testBuilder()
+ .sqlQuery("SELECT * FROM " +
+ "(SELECT CAST(COUNT(employee_id) AS BIGINT) as c1, " +
+ "CAST(SUM(employee_id) AS INT) as s1, " +
+ "CAST(COUNT(DISTINCT employee_id) AS BIGINT) as c2 " +
+ "FROM cp.`employee.json`) " +
+ "T LIMIT 0")
+ .baselineColumns("c1", "s1", "c2")
+ .expectsEmptyResultSet()
+ .go();
+ }
+
+ @Test
+ public void countDistinctLimit0Plan() throws Exception {
+ PlanTestBase.testPlanMatchingPatterns("SELECT * FROM " +
+ "(SELECT CAST(COUNT(employee_id) AS BIGINT) as c1, " +
+ "CAST(SUM(employee_id) AS INT) as s1, " +
+ "CAST(COUNT(DISTINCT employee_id) AS BIGINT) as c2 " +
+ "FROM cp.`employee.json`) " +
+ "T LIMIT 0",
+ new String[] {
+ ".*Limit.*\n" +
+ ".*Values.*"
+ },
+ new String[] {});
+ }
+}
diff --git
a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2288GetColumnsMetadataWhenNoRowsTest.java
b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2288GetColumnsMetadataWhenNoRowsTest.java
index 0455086a5b..ac3523e629 100644
---
a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2288GetColumnsMetadataWhenNoRowsTest.java
+++
b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/Drill2288GetColumnsMetadataWhenNoRowsTest.java
@@ -56,6 +56,19 @@ public static void tearDownConnection() throws SQLException {
connection.close();
}
+ protected void checkForSchemaAndNoRows(final String query) throws Exception {
+ try (final Statement stmt = connection.createStatement();
+ final ResultSet results = stmt.executeQuery(query)) {
+
+ // Result set should still have columns even though there are no rows:
+ ResultSetMetaData metadata = results.getMetaData();
+ assertThat("ResultSetMetaData.getColumnCount() should have been > 0",
+ metadata.getColumnCount(), not(equalTo(0)));
+
+ assertThat("Unexpected non-empty results. Test rot?",
+ false, equalTo(results.next()));
+ }
+ }
/**
* Tests that an empty JSON file (having zero records) no longer triggers
@@ -64,16 +77,7 @@ public static void tearDownConnection() throws SQLException {
*/
@Test
public void testEmptyJsonFileDoesntSuppressNetSchema1() throws Exception {
- Statement stmt = connection.createStatement();
- ResultSet results = stmt.executeQuery( "SELECT a, b, c, * FROM
cp.`empty.json`" );
-
- // Result set should still have columns even though there are no rows:
- ResultSetMetaData metadata = results.getMetaData();
- assertThat( "ResultSetMetaData.getColumnCount() should have been > 0",
- metadata.getColumnCount(), not( equalTo( 0 ) ) );
-
- assertThat( "Unexpected non-empty results. Test rot?",
- false, equalTo( results.next() ) );
+ checkForSchemaAndNoRows("SELECT a, b, c, * FROM cp.`empty.json`");
}
@Test
@@ -98,87 +102,32 @@ public void testEmptyJsonFileDoesntSuppressNetSchema2()
throws Exception {
*/
@Test
public void testInfoSchemaTablesZeroRowsBy_TABLE_SCHEMA_works() throws
Exception {
- Statement stmt = connection.createStatement();
- ResultSet results =
- stmt.executeQuery( "SELECT * FROM INFORMATION_SCHEMA.`TABLES`"
- + " WHERE TABLE_SCHEMA = ''" );
-
- // Result set should still have columns even though there are no rows:
- ResultSetMetaData metadata = results.getMetaData();
- assertThat( "ResultSetMetaData.getColumnCount() should have been > 0",
- metadata.getColumnCount(), not( equalTo( 0 ) ) );
-
- assertThat( "Unexpected non-empty results. Test rot?",
- false, equalTo( results.next() ) );
+ checkForSchemaAndNoRows("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE
TABLE_SCHEMA = ''");
}
/** (Worked before (because TABLE_CATALOG test not pushed down).) */
@Test
public void testInfoSchemaTablesZeroRowsBy_TABLE_CATALOG_works() throws
Exception {
- Statement stmt = connection.createStatement();
- ResultSet results =
- stmt.executeQuery( "SELECT * FROM INFORMATION_SCHEMA.`TABLES`"
- + " WHERE TABLE_CATALOG = ''" );
-
- // Result set should still have columns even though there are no rows:
- ResultSetMetaData metadata = results.getMetaData();
- assertThat( "ResultSetMetaData.getColumnCount() should have been > 0",
- metadata.getColumnCount(), not( equalTo( 0 ) ) );
-
- assertThat( "Unexpected non-empty results. Test rot?",
- false, equalTo( results.next() ) );
+ checkForSchemaAndNoRows("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE
TABLE_CATALOG = ''");
}
/** (Failed before (because TABLE_NAME test is pushed down).) */
@Test
public void testInfoSchemaTablesZeroRowsBy_TABLE_NAME_works()
throws Exception {
- Statement stmt = connection.createStatement();
- ResultSet results =
- stmt.executeQuery(
- "SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE TABLE_NAME = ''"
);
-
- // Result set should still have columns even though there are no rows:
- ResultSetMetaData metadata = results.getMetaData();
- assertThat( "ResultSetMetaData.getColumnCount() should have been > 0",
- metadata.getColumnCount(), not( equalTo( 0 ) ) );
-
- assertThat( "Unexpected non-empty results. Test rot?",
- false, equalTo( results.next() ) );
+ checkForSchemaAndNoRows("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE
TABLE_NAME = ''");
}
/** (Worked before.) */
@Test
public void testInfoSchemaTablesZeroRowsByLimitWorks() throws Exception {
- Statement stmt = connection.createStatement();
- ResultSet results =
- stmt.executeQuery(
- "SELECT * FROM INFORMATION_SCHEMA.`TABLES` LIMIT 0" );
-
- // Result set should still have columns even though there are no rows:
- ResultSetMetaData metadata = results.getMetaData();
- assertThat( "ResultSetMetaData.getColumnCount() should have been > 0",
- metadata.getColumnCount(), not( equalTo( 0 ) ) );
-
- assertThat( "Unexpected non-empty results. Test rot?",
- false, equalTo( results.next() ) );
+ checkForSchemaAndNoRows("SELECT * FROM INFORMATION_SCHEMA.`TABLES` LIMIT
0");
}
/** (Worked before.) */
@Test
public void testInfoSchemaTablesZeroRowsByWhereFalseWorks() throws Exception
{
- Statement stmt = connection.createStatement();
- ResultSet results =
- stmt.executeQuery(
- "SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE FALSE" );
-
- // Result set should still have columns even though there are no rows:
- ResultSetMetaData metadata = results.getMetaData();
- assertThat( "ResultSetMetaData.getColumnCount() should have been > 0",
- metadata.getColumnCount(), not( equalTo( 0 ) ) );
-
- assertThat( "Unexpected non-empty results. Test rot?",
- false, equalTo( results.next() ) );
+ checkForSchemaAndNoRows("SELECT * FROM INFORMATION_SCHEMA.`TABLES` WHERE
FALSE");
}
/** (Failed before (because table schema and name tests are pushed down).) */
@@ -197,5 +146,4 @@ public void testGetTablesZeroRowsByTableSchemaOrNameWorks()
throws Exception {
false, equalTo( results.next() ) );
}
-
}
diff --git a/pom.xml b/pom.xml
index 0f0053d6bb..86588dbfe8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.drill</groupId>
<artifactId>drill-root</artifactId>
- <version>1.5.0-SNAPSHOT</version>
+ <version>1.5.0-limit</version>
<packaging>pom</packaging>
<name>Apache Drill Root POM</name>
@@ -1278,7 +1278,7 @@
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
- <version>1.4.0-drill-r10</version>
+ <version>1.4.0-drill-1.4.0-mapr-r1</version>
<exclusions>
<exclusion>
<groupId>org.jgrapht</groupId>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Limit 0 should avoid execution when querying a known schema
> -----------------------------------------------------------
>
> Key: DRILL-3623
> URL: https://issues.apache.org/jira/browse/DRILL-3623
> Project: Apache Drill
> Issue Type: Sub-task
> Components: Storage - Hive
> Affects Versions: 1.1.0
> Environment: MapR cluster
> Reporter: Andries Engelbrecht
> Assignee: Sudheesh Katkam
> Priority: Major
> Labels: doc-impacting
> Fix For: 1.7.0
>
>
> Running a select * from hive.table limit 0 does not return (hangs).
> Select * from hive.table limit 1 works fine
> Hive table is about 6GB with 330 files with parquet using snappy compression.
> Data types are int, bigint, string and double.
> Querying directory with parquet files through the DFS plugin works fine
> select * from dfs.root.`/user/hive/warehouse/database/table` limit 0;
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
