Repository: calcite Updated Branches: refs/heads/master 051809b4d -> a3dd90f10
[CALCITE-2016] ITEM + DOT operator does not work for array (Shuyi Chen) Add parser rules to support chaining ITEM and DOT operators for complex types. Close apache/calcite#557 Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/a3dd90f1 Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/a3dd90f1 Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/a3dd90f1 Branch: refs/heads/master Commit: a3dd90f1088966dc09caecf88779853ba34d290c Parents: e8481e5 Author: Shuyi Chen <[email protected]> Authored: Sun Nov 5 23:09:02 2017 -0800 Committer: Julian Hyde <[email protected]> Committed: Wed Nov 15 22:45:49 2017 -0800 ---------------------------------------------------------------------- core/src/main/codegen/templates/Parser.jj | 10 ++ .../apache/calcite/sql/fun/SqlDotOperator.java | 169 +++++++++++++++++++ .../calcite/sql/fun/SqlStdOperatorTable.java | 10 +- .../calcite/sql/validate/SqlValidatorImpl.java | 13 +- .../sql2rel/StandardConvertletTable.java | 10 ++ .../calcite/sql/parser/SqlParserTest.java | 5 + .../apache/calcite/test/MockCatalogReader.java | 9 +- .../calcite/test/SqlToRelConverterTest.java | 4 + .../apache/calcite/test/SqlValidatorTest.java | 15 +- .../calcite/test/SqlToRelConverterTest.xml | 11 ++ 10 files changed, 241 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/main/codegen/templates/Parser.jj ---------------------------------------------------------------------- diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index fb0439b..f71abce 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -2974,6 +2974,7 @@ List<Object> Expression2(ExprContext exprContext) : SqlNodeList nodeList; SqlNode e; SqlOperator op; + String p; final Span s = span(); } { @@ -3095,6 +3096,15 @@ List<Object> Expression2(ExprContext exprContext) : SqlStdOperatorTable.ITEM, getPos())); list.add(e); } + ( + <DOT> + p = Identifier() { + list.add( + new SqlParserUtil.ToTreeListItem( + SqlStdOperatorTable.DOT, getPos())); + list.add(new SqlIdentifier(p, getPos())); + } + )* | { checkNonQueryExpression(exprContext); http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java new file mode 100644 index 0000000..45e9bd9 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlDotOperator.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql.fun; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlUtil; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlSingleOperandTypeChecker; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.apache.calcite.sql.util.SqlVisitor; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.calcite.sql.validate.SqlValidatorUtil; +import org.apache.calcite.util.Static; + +import java.util.Arrays; + +/** + * The dot operator {@code .}, used to access a field of a + * record. For example, {@code a.b}. + */ +public class SqlDotOperator extends SqlSpecialOperator { + SqlDotOperator() { + super("DOT", SqlKind.DOT, 100, true, null, null, null); + } + + @Override public ReduceResult reduceExpr(int ordinal, TokenSequence list) { + SqlNode left = list.node(ordinal - 1); + SqlNode right = list.node(ordinal + 1); + return new ReduceResult(ordinal - 1, + ordinal + 2, + createCall( + SqlParserPos.sum( + Arrays.asList(left.getParserPosition(), + right.getParserPosition(), + list.pos(ordinal))), + left, + right)); + } + + @Override public void unparse(SqlWriter writer, SqlCall call, int leftPrec, + int rightPrec) { + final SqlWriter.Frame frame = + writer.startList(SqlWriter.FrameTypeEnum.IDENTIFIER); + call.operand(0).unparse(writer, leftPrec, 0); + writer.sep("."); + call.operand(1).unparse(writer, 0, 0); + writer.endList(frame); + } + + @Override public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.of(2); + } + + @Override public <R> void acceptCall(SqlVisitor<R> visitor, SqlCall call, + boolean onlyExpressions, SqlBasicVisitor.ArgHandler<R> argHandler) { + // Do not visit operands[1] here. + argHandler.visitChild(visitor, call, 0, call.operand(0)); + } + + @Override public RelDataType deriveType(SqlValidator validator, + SqlValidatorScope scope, SqlCall call) { + RelDataType nodeType = validator.deriveType(scope, call.getOperandList().get(0)); + assert nodeType != null; + + final String fieldName = call.getOperandList().get(1).toString(); + RelDataTypeField field = + nodeType.getField(fieldName, false, false); + if (field == null) { + throw SqlUtil.newContextException(SqlParserPos.ZERO, Static.RESOURCE.unknownField(fieldName)); + } + RelDataType type = field.getType(); + + // Validate and determine coercibility and resulting collation + // name of binary operator if needed. + type = adjustType(validator, call, type); + SqlValidatorUtil.checkCharsetAndCollateConsistentIfCharType(type); + return type; + } + + public void validateCall( + SqlCall call, + SqlValidator validator, + SqlValidatorScope scope, + SqlValidatorScope operandScope) { + assert call.getOperator() == this; + // Do not visit call.getOperandList().get(1) here. + // call.getOperandList().get(1) will be validated when deriveType() is called. + call.getOperandList().get(0).validateExpr(validator, operandScope); + } + + @Override public boolean checkOperandTypes(SqlCallBinding callBinding, + boolean throwOnFailure) { + final SqlNode left = callBinding.operand(0); + final SqlNode right = callBinding.operand(1); + final RelDataType type = + callBinding.getValidator().deriveType(callBinding.getScope(), left); + if (type.getSqlTypeName() != SqlTypeName.ROW) { + return false; + } + final RelDataType operandType = callBinding.getOperandType(0); + final SqlSingleOperandTypeChecker checker = getChecker(operandType); + return checker.checkSingleOperandType(callBinding, right, 0, + throwOnFailure); + } + + private SqlSingleOperandTypeChecker getChecker(RelDataType operandType) { + switch (operandType.getSqlTypeName()) { + case ROW: + return OperandTypes.family(SqlTypeFamily.STRING); + default: + throw new AssertionError(operandType.getSqlTypeName()); + } + } + + @Override public String getAllowedSignatures(String name) { + return "<A>.<B>"; + } + + @Override public RelDataType inferReturnType(SqlOperatorBinding opBinding) { + final RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + final RelDataType recordType = opBinding.getOperandType(0); + switch (recordType.getSqlTypeName()) { + case ROW: + final String fieldName = + opBinding.getOperandLiteralValue(1, String.class); + final RelDataType type = opBinding.getOperandType(0) + .getField(fieldName, false, false) + .getType(); + if (recordType.isNullable()) { + return typeFactory.createTypeWithNullability(type, true); + } else { + return type; + } + default: + throw new AssertionError(); + } + } +} + +// End SqlDotOperator.java http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index f6a762b..c56b8bc 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -295,15 +295,7 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { /** * Dot operator, '<code>.</code>', used for referencing fields of records. */ - public static final SqlBinaryOperator DOT = - new SqlBinaryOperator( - ".", - SqlKind.DOT, - 80, - true, - null, - null, - OperandTypes.ANY_ANY); + public static final SqlOperator DOT = new SqlDotOperator(); /** * Logical equals operator, '<code>=</code>'. http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 90bdb88..940c8ac 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -3273,6 +3273,13 @@ public class SqlValidatorImpl implements SqlValidatorWithHints { } } + private SqlNode stripDot(SqlNode node) { + if (node != null && node.getKind() == SqlKind.DOT) { + return ((SqlCall) node).operand(0); + } + return node; + } + private void checkRollUp(SqlNode grandParent, SqlNode parent, SqlNode current, SqlValidatorScope scope, String optionalClause) { current = stripAs(current); @@ -3281,7 +3288,7 @@ public class SqlValidatorImpl implements SqlValidatorWithHints { checkRollUpInWindow(getWindowInOver(current), scope); current = stripOver(current); - List<SqlNode> children = ((SqlCall) current).getOperandList(); + List<SqlNode> children = ((SqlCall) stripDot(current)).getOperandList(); for (SqlNode child : children) { checkRollUp(parent, current, child, scope, optionalClause); } @@ -3289,10 +3296,10 @@ public class SqlValidatorImpl implements SqlValidatorWithHints { SqlIdentifier id = (SqlIdentifier) current; if (!id.isStar() && isRolledUpColumn(id, scope)) { if (!isAggregation(parent.getKind()) - || !isRolledUpColumnAllowedInAgg(id, scope, (SqlCall) parent, grandParent)) { + || !isRolledUpColumnAllowedInAgg(id, scope, (SqlCall) parent, grandParent)) { String context = optionalClause != null ? optionalClause : parent.getKind().toString(); throw newValidationError(id, - RESOURCE.rolledUpNotAllowed(deriveAlias(id, 0), context)); + RESOURCE.rolledUpNotAllowed(deriveAlias(id, 0), context)); } } } http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java index 4c45716..158ffa6 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/StandardConvertletTable.java @@ -248,6 +248,16 @@ public class StandardConvertletTable extends ReflectiveConvertletTable { } }); + // "DOT" + registerOp( + SqlStdOperatorTable.DOT, + new SqlRexConvertlet() { + public RexNode convertCall(SqlRexContext cx, SqlCall call) { + return cx.getRexBuilder().makeCall(SqlStdOperatorTable.DOT, + cx.convertExpression(call.operand(0)), + cx.getRexBuilder().makeLiteral(call.operand(1).toString())); + } + }); // "AS" has no effect, so expand "x AS id" into "x". registerOp( SqlStdOperatorTable.AS, http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java index 1042e20..4f2d9cb 100644 --- a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -4250,6 +4250,11 @@ public class SqlParserTest { checkExp("a[b[1 + 2] + 3]", "`A`[(`B`[(1 + 2)] + 3)]"); } + @Test public void testArrayElementWithDot() { + checkExp("a[1+2].b.c[2].d", "(((`A`[(1 + 2)].`B`).`C`)[2].`D`)"); + checkExp("a[b[1]].c.f0[d[1]]", "((`A`[`B`[1]].`C`).`F0`)[`D`[1]]"); + } + @Test public void testArrayValueConstructor() { checkExp("array[1, 2]", "(ARRAY[1, 2])"); checkExp("array [1, 2]", "(ARRAY[1, 2])"); // with space http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/test/java/org/apache/calcite/test/MockCatalogReader.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/test/MockCatalogReader.java b/core/src/test/java/org/apache/calcite/test/MockCatalogReader.java index ed1f952..b25ff10 100644 --- a/core/src/test/java/org/apache/calcite/test/MockCatalogReader.java +++ b/core/src/test/java/org/apache/calcite/test/MockCatalogReader.java @@ -1615,10 +1615,17 @@ public class MockCatalogReader extends CalciteCatalogReader { .build()) .kind(StructKind.PEEK_FIELDS_NO_EXPAND) .build(); + final RelDataType skillRecordType = + typeFactory.builder() + .add("TYPE", varchar10Type) + .add("DESC", varchar20Type) + .build(); final RelDataType empRecordType = typeFactory.builder() .add("EMPNO", intType) - .add("ENAME", varchar10Type).build(); + .add("ENAME", varchar10Type) + .add("SKILLS", typeFactory.createArrayType(skillRecordType, -1)) + .build(); final RelDataType empListType = typeFactory.createArrayType(empRecordType, -1); http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index aebcece..dd46878 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -1033,6 +1033,10 @@ public class SqlToRelConverterTest extends SqlToRelTestBase { sql(sql).ok(); } + @Test public void testArrayOfRecord() { + sql("select employees[1].skills[2+3].desc from dept_nested").ok(); + } + @Test public void testUnnestArray() { sql("select*from unnest(array(select*from dept))").ok(); } http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 570c7a0..49de12d 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -7400,7 +7400,9 @@ public class SqlValidatorTest extends SqlValidatorTestCase { + " UNNEST(d.employees) as e"; final String type = "RecordType(INTEGER NOT NULL DEPTNO," + " INTEGER NOT NULL EMPNO," - + " VARCHAR(10) NOT NULL ENAME) NOT NULL"; + + " VARCHAR(10) NOT NULL ENAME," + + " RecordType(VARCHAR(10) NOT NULL TYPE, VARCHAR(20) NOT NULL DESC)" + + " NOT NULL ARRAY NOT NULL SKILLS) NOT NULL"; sql(sql).type(type); // equivalent query using CROSS JOIN @@ -7777,6 +7779,15 @@ public class SqlValidatorTest extends SqlValidatorTestCase { "RecordType(INTEGER NOT NULL X, VARCHAR(20) NOT NULL EMAIL, INTEGER NOT NULL Y) NOT NULL"); } + @Test public void testArrayOfRecordType() { + sql("SELECT name, dept_nested.employees[1].ne as ne from dept_nested") + .fails("Unknown field 'NE'"); + sql("SELECT name, dept_nested.employees[1].ename as ename from dept_nested") + .type("RecordType(VARCHAR(10) NOT NULL NAME, VARCHAR(10) ENAME) NOT NULL"); + sql("SELECT dept_nested.employees[1].skills[1].desc as DESCRIPTION from dept_nested") + .type("RecordType(VARCHAR(20) DESCRIPTION) NOT NULL"); + } + /** Test case for * <a href="https://issues.apache.org/jira/browse/CALCITE-497";>[CALCITE-497] * Support optional qualifier for column name references</a>. */ @@ -8465,6 +8476,7 @@ public class SqlValidatorTest extends SqlValidatorTestCase { + "\n" + "CURRENT_VALUE -\n" + "DEFAULT -\n" + + "DOT -\n" + "ITEM -\n" + "NEXT_VALUE -\n" + "PATTERN_EXCLUDE -\n" @@ -8476,7 +8488,6 @@ public class SqlValidatorTest extends SqlValidatorTestCase { + "$LiteralChain -\n" + "+ pre\n" + "- pre\n" - + ". left\n" + "FINAL pre\n" + "RUNNING pre\n" + "\n" http://git-wip-us.apache.org/repos/asf/calcite/blob/a3dd90f1/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 6818e63..34037ba 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -359,6 +359,17 @@ from orders window w as (partition by productId)]]> </Resource> </TestCase> + <TestCase name="testArrayOfRecord"> + <Resource name="sql"> + <![CDATA[select*from unnest(array(select*from dept))]]> + </Resource> + <Resource name="plan"> + <![CDATA[ +LogicalProject(EXPR$0=[DOT(ITEM(DOT(ITEM($2, 1), 'SKILLS'), +(2, 3)), 'DESC')]) + LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED]]) +]]> + </Resource> + </TestCase> <TestCase name="testUnnestArray"> <Resource name="sql"> <![CDATA[select*from unnest(array(select*from dept))]]>
