Repository: tajo Updated Branches: refs/heads/master 550c0189b -> 492588d62
TAJO-1922: Allow date types with IN operator for partition pruning. Closes #819 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/492588d6 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/492588d6 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/492588d6 Branch: refs/heads/master Commit: 492588d62e1f1300854e51c61b79c4e875d7f15a Parents: 550c018 Author: JaeHwa Jung <[email protected]> Authored: Thu Nov 26 13:57:43 2015 +0900 Committer: JaeHwa Jung <[email protected]> Committed: Thu Nov 26 13:57:43 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../tajo/engine/query/TestTablePartitions.java | 68 +++++++++++++++ .../tajo/plan/util/EvalNodeToExprConverter.java | 88 +++++++++----------- 3 files changed, 109 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/492588d6/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 00b9129..41b4552 100644 --- a/CHANGES +++ b/CHANGES @@ -88,6 +88,8 @@ Release 0.12.0 - unreleased TASKS + TAJO-1922: Allow date types with IN operator for partition pruning. (jaehwa) + TAJO-1857: Rename the section of 'File Formats' to 'Data Formats' and fill compression section of the 'Table Management' chapter. (Contributed by Jongyoung Park. Committed by jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/492588d6/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index b4bc8ee..de2cbdf 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -1837,4 +1837,72 @@ public class TestTablePartitions extends QueryTestCaseBase { executeString("DROP TABLE " + externalTableName).close(); executeString("DROP TABLE " + tableName + " PURGE").close(); } + + @Test + public final void testPartitionWithInOperator() throws Exception { + ResultSet res = null; + String tableName = CatalogUtil.normalizeIdentifier("testPartitionWithInOperator"); + String result, expectedResult; + + if (nodeType == NodeType.INSERT) { + res = testBase.execute( + "create table " + tableName + " (col4 text) partition by column(col1 int4, col2 int4, col3 float8) "); + res.close(); + TajoTestingCluster cluster = testBase.getTestingCluster(); + CatalogService catalog = cluster.getMaster().getCatalog(); + assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); + + res = executeString("insert overwrite into " + tableName + + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem"); + } else { + res = executeString( "create table " + tableName + " (col4 text) " + + " partition by column(col1 int4, col2 int4, col3 float8) as select l_returnflag, l_orderkey, l_partkey, " + + "l_quantity from lineitem"); + } + res.close(); + + res = executeString("select * from " + tableName); + result = resultSetToString(res); + expectedResult = "col4,col1,col2,col3\n" + + "-------------------------------\n" + + "N,1,1,17.0\n" + + "N,1,1,36.0\n" + + "N,2,2,38.0\n" + + "R,3,2,45.0\n" + + "R,3,3,49.0\n"; + res.close(); + assertEquals(expectedResult, result); + + res = executeString("select * from " + tableName + " WHERE col1 in (1, 2) order by col3"); + result = resultSetToString(res); + expectedResult = "col4,col1,col2,col3\n" + + "-------------------------------\n" + + "N,1,1,17.0\n" + + "N,1,1,36.0\n" + + "N,2,2,38.0\n"; + res.close(); + assertEquals(expectedResult, result); + + res = executeString("select * from " + tableName + " WHERE col1 in (2, 3) and col2 = 2 order by col3"); + result = resultSetToString(res); + expectedResult = "col4,col1,col2,col3\n" + + "-------------------------------\n" + + "N,2,2,38.0\n" + + "R,3,2,45.0\n"; + res.close(); + assertEquals(expectedResult, result); + + res = executeString("select * from " + tableName + " WHERE col1 in (1, 2, 3) and col2 in (1, 3) and col3 < 30 " + + " order by col3"); + result = resultSetToString(res); + expectedResult = "col4,col1,col2,col3\n" + + "-------------------------------\n" + + "N,1,1,17.0\n"; + res.close(); + assertEquals(expectedResult, result); + + executeString("DROP TABLE " + tableName + " PURGE").close(); + res.close(); + } + } http://git-wip-us.apache.org/repos/asf/tajo/blob/492588d6/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java index 139bcc0..00638c2 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java @@ -19,10 +19,12 @@ package org.apache.tajo.plan.util; import org.apache.tajo.algebra.*; +import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.DateDatum; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.TimeDatum; import org.apache.tajo.datum.TimestampDatum; +import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.expr.*; import java.util.Stack; @@ -143,34 +145,59 @@ public class EvalNodeToExprConverter extends SimpleEvalNodeVisitor<Object> { @Override protected EvalNode visitConst(Object o, ConstEval evalNode, Stack<EvalNode> stack) { + exprs.push(convertDatumToExpr(evalNode.getValueType().getType(), evalNode.getValue())); + return super.visitConst(o, evalNode, stack); + } + + @Override + protected EvalNode visitRowConstant(Object o, RowConstantEval evalNode, Stack<EvalNode> stack) { + Expr[] values = new Expr[evalNode.getValues().length]; + for (int i = 0; i < evalNode.getValues().length; i++) { + Datum datum = evalNode.getValues()[i]; + values[i] = convertDatumToExpr(datum.type(), datum); + } + ValueListExpr expr = new ValueListExpr(values); + exprs.push(expr); + + return super.visitRowConstant(o, evalNode, stack); + } + + /** + * Convert specified Datum to Expr + * + * @param type value type + * @param datum target datum + * @return converted datum + */ + private Expr convertDatumToExpr(TajoDataTypes.Type type, Datum datum) { Expr value = null; DateValue dateValue; TimeValue timeValue; - switch (evalNode.getValueType().getType()) { + switch (type) { case NULL_TYPE: value = new NullLiteral(); break; case BOOLEAN: - value = new LiteralValue(evalNode.getValue().asChars(), LiteralValue.LiteralType.Boolean); + value = new BooleanLiteral(datum.asBool()); break; case INT1: case INT2: case INT4: - value = new LiteralValue(evalNode.getValue().asChars(), LiteralValue.LiteralType.Unsigned_Integer); + value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.Unsigned_Integer); break; case INT8: - value = new LiteralValue(evalNode.getValue().asChars(), LiteralValue.LiteralType.Unsigned_Large_Integer); + value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.Unsigned_Large_Integer); break; case FLOAT4: case FLOAT8: - value = new LiteralValue(evalNode.getValue().asChars(), LiteralValue.LiteralType.Unsigned_Float); + value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.Unsigned_Float); break; case TEXT: - value = new LiteralValue(evalNode.getValue().asChars(), LiteralValue.LiteralType.String); + value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.String); break; case DATE: - DateDatum dateDatum = (DateDatum) evalNode.getValue(); + DateDatum dateDatum = (DateDatum) datum; dateValue = new DateValue(""+dateDatum.getYear(), ""+dateDatum.getMonthOfYear(), ""+dateDatum.getDayOfMonth()); @@ -178,65 +205,28 @@ public class EvalNodeToExprConverter extends SimpleEvalNodeVisitor<Object> { break; case TIMESTAMP: - TimestampDatum timestampDatum = (TimestampDatum) evalNode.getValue(); + TimestampDatum timestampDatum = (TimestampDatum) datum; dateValue = new DateValue(""+timestampDatum.getYear(), ""+timestampDatum.getMonthOfYear(), ""+timestampDatum.getDayOfMonth()); timeValue = new TimeValue(""+timestampDatum.getHourOfDay() - , ""+timestampDatum.getMinuteOfHour(), ""+timestampDatum.getSecondOfMinute()); + , ""+timestampDatum.getMinuteOfHour(), ""+timestampDatum.getSecondOfMinute()); value = new TimestampLiteral(dateValue, timeValue); break; case TIME: - TimeDatum timeDatum = (TimeDatum) evalNode.getValue(); + TimeDatum timeDatum = (TimeDatum) datum; timeValue = new TimeValue(""+timeDatum.getHourOfDay() , ""+timeDatum.getMinuteOfHour(), ""+timeDatum.getSecondOfMinute()); value = new TimeLiteral(timeValue); break; default: - throw new RuntimeException("Unsupported type: " + evalNode.getValueType().getType().name()); - } - exprs.push(value); - - return super.visitConst(o, evalNode, stack); - } - - @Override - protected EvalNode visitRowConstant(Object o, RowConstantEval evalNode, Stack<EvalNode> stack) { - Expr[] values = new Expr[evalNode.getValues().length]; - for (int i = 0; i < evalNode.getValues().length; i++) { - Datum datum = evalNode.getValues()[i]; - LiteralValue value; - switch (datum.type()) { - case BOOLEAN: - value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.Boolean); - break; - case TEXT: - value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.String); - break; - case INT1: - case INT2: - case INT4: - value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.Unsigned_Integer); - break; - case INT8: - value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.Unsigned_Large_Integer); - break; - case FLOAT4: - case FLOAT8: - value = new LiteralValue(datum.asChars(), LiteralValue.LiteralType.Unsigned_Float); - break; - default: - throw new RuntimeException("Unsupported type: " + datum.type().name()); - } - values[i] = value; + throw new RuntimeException(new UnsupportedException(type.name())); } - ValueListExpr expr = new ValueListExpr(values); - exprs.push(expr); - return super.visitRowConstant(o, evalNode, stack); + return value; } @Override
