http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/ArithmeticExpr.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/ArithmeticExpr.java b/fe/src/main/java/org/apache/impala/analysis/ArithmeticExpr.java new file mode 100644 index 0000000..bf8b0ea --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/ArithmeticExpr.java @@ -0,0 +1,268 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.catalog.Db; +import com.cloudera.impala.catalog.Function.CompareMode; +import com.cloudera.impala.catalog.ScalarFunction; +import com.cloudera.impala.catalog.ScalarType; +import com.cloudera.impala.catalog.Type; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TExprNode; +import com.cloudera.impala.thrift.TExprNodeType; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +public class ArithmeticExpr extends Expr { + enum OperatorPosition { + BINARY_INFIX, + UNARY_PREFIX, + UNARY_POSTFIX, + } + + enum Operator { + MULTIPLY("*", "multiply", OperatorPosition.BINARY_INFIX), + DIVIDE("/", "divide", OperatorPosition.BINARY_INFIX), + MOD("%", "mod", OperatorPosition.BINARY_INFIX), + INT_DIVIDE("DIV", "int_divide", OperatorPosition.BINARY_INFIX), + ADD("+", "add", OperatorPosition.BINARY_INFIX), + SUBTRACT("-", "subtract", OperatorPosition.BINARY_INFIX), + BITAND("&", "bitand", OperatorPosition.BINARY_INFIX), + BITOR("|", "bitor", OperatorPosition.BINARY_INFIX), + BITXOR("^", "bitxor", OperatorPosition.BINARY_INFIX), + BITNOT("~", "bitnot", OperatorPosition.UNARY_PREFIX), + FACTORIAL("!", "factorial", OperatorPosition.UNARY_POSTFIX); + + private final String description_; + private final String name_; + private final OperatorPosition pos_; + + private Operator(String description, String name, OperatorPosition pos) { + this.description_ = description; + this.name_ = name; + this.pos_ = pos; + } + + @Override + public String toString() { return description_; } + public String getName() { return name_; } + public OperatorPosition getPos() { return pos_; } + + public boolean isUnary() { + return pos_ == OperatorPosition.UNARY_PREFIX || + pos_ == OperatorPosition.UNARY_POSTFIX; + } + + public boolean isBinary() { + return pos_ == OperatorPosition.BINARY_INFIX; + } + } + + private final Operator op_; + + public Operator getOp() { return op_; } + + public ArithmeticExpr(Operator op, Expr e1, Expr e2) { + super(); + this.op_ = op; + Preconditions.checkNotNull(e1); + children_.add(e1); + Preconditions.checkArgument((op.isUnary() && e2 == null) || + (op.isBinary() && e2 != null)); + if (e2 != null) children_.add(e2); + } + + /** + * Copy c'tor used in clone(). + */ + protected ArithmeticExpr(ArithmeticExpr other) { + super(other); + op_ = other.op_; + } + + public static void initBuiltins(Db db) { + for (Type t: Type.getNumericTypes()) { + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.MULTIPLY.getName(), Lists.newArrayList(t, t), t)); + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.ADD.getName(), Lists.newArrayList(t, t), t)); + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.SUBTRACT.getName(), Lists.newArrayList(t, t), t)); + } + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.DIVIDE.getName(), + Lists.<Type>newArrayList(Type.DOUBLE, Type.DOUBLE), + Type.DOUBLE)); + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.DIVIDE.getName(), + Lists.<Type>newArrayList(Type.DECIMAL, Type.DECIMAL), + Type.DECIMAL)); + + /* + * MOD(), FACTORIAL(), BITAND(), BITOR(), BITXOR(), and BITNOT() are registered as + * builtins, see impala_functions.py + */ + for (Type t: Type.getIntegerTypes()) { + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.INT_DIVIDE.getName(), Lists.newArrayList(t, t), t)); + } + } + + @Override + public String debugString() { + return Objects.toStringHelper(this) + .add("op", op_) + .addValue(super.debugString()) + .toString(); + } + + @Override + public String toSqlImpl() { + if (children_.size() == 1) { + if (op_.getPos() == OperatorPosition.UNARY_PREFIX) { + return op_.toString() + getChild(0).toSql(); + } else { + assert(op_.getPos() == OperatorPosition.UNARY_POSTFIX); + return getChild(0).toSql() + op_.toString(); + } + } else { + Preconditions.checkState(children_.size() == 2); + return getChild(0).toSql() + " " + op_.toString() + " " + getChild(1).toSql(); + } + } + + @Override + protected void toThrift(TExprNode msg) { + msg.node_type = TExprNodeType.FUNCTION_CALL; + } + + /** + * Inserts a cast from child[childIdx] to targetType if one is necessary. + * Note this is different from Expr.castChild() since arithmetic for decimals + * the cast is handled as part of the operator and in general, the return type + * does not match the input types. + */ + void castChild(int childIdx, Type targetType) throws AnalysisException { + Type t = getChild(childIdx).getType(); + if (t.matchesType(targetType)) return; + if (targetType.isDecimal() && !t.isNull()) { + Preconditions.checkState(t.isScalarType()); + targetType = ((ScalarType) t).getMinResolutionDecimal(); + } + castChild(targetType, childIdx); + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (isAnalyzed_) return; + super.analyze(analyzer); + for (Expr child: children_) { + Expr operand = (Expr) child; + if (!operand.type_.isNumericType() && !operand.type_.isNull()) { + String errMsg = "Arithmetic operation requires numeric operands: " + toSql(); + if (operand instanceof Subquery && !operand.type_.isScalarType()) { + errMsg = "Subquery must return a single row: " + operand.toSql(); + } + throw new AnalysisException(errMsg); + } + } + + convertNumericLiteralsFromDecimal(analyzer); + Type t0 = getChild(0).getType(); + Type t1 = null; + if (op_.isUnary()) { + Preconditions.checkState(children_.size() == 1); + } else if (op_.isBinary()) { + Preconditions.checkState(children_.size() == 2); + t1 = getChild(1).getType(); + } + if (hasChildCosts()) evalCost_ = getChildCosts() + ARITHMETIC_OP_COST; + + String fnName = op_.getName(); + switch (op_) { + case ADD: + case SUBTRACT: + case DIVIDE: + case MULTIPLY: + case MOD: + type_ = TypesUtil.getArithmeticResultType(t0, t1, op_); + // If both of the children are null, we'll default to the DOUBLE version of the + // operator. This prevents the BE from seeing NULL_TYPE. + if (type_.isNull()) type_ = Type.DOUBLE; + break; + + case INT_DIVIDE: + case BITAND: + case BITOR: + case BITXOR: + if ((!t0.isNull() & !t0.isIntegerType()) || + (!t1.isNull() && !t1.isIntegerType())) { + throw new AnalysisException("Invalid non-integer argument to operation '" + + op_.toString() + "': " + this.toSql()); + } + type_ = Type.getAssignmentCompatibleType(t0, t1, false); + // If both of the children are null, we'll default to the INT version of the + // operator. This prevents the BE from seeing NULL_TYPE. + if (type_.isNull()) type_ = Type.INT; + Preconditions.checkState(type_.isIntegerType()); + break; + case BITNOT: + case FACTORIAL: + if (!t0.isNull() && !t0.isIntegerType()) { + throw new AnalysisException("'" + op_.toString() + "'" + + " operation only allowed on integer types: " + toSql()); + } + // Special-case NULL to resolve to the appropriate type. + if (op_ == Operator.BITNOT) { + if (t0.isNull()) castChild(0, Type.INT); + } else { + assert(op_ == Operator.FACTORIAL); + if (t0.isNull()) castChild(0, Type.BIGINT); + } + fn_ = getBuiltinFunction(analyzer, op_.getName(), collectChildReturnTypes(), + CompareMode.IS_SUPERTYPE_OF); + Preconditions.checkNotNull(fn_); + castForFunctionCall(false); + type_ = fn_.getReturnType(); + return; + default: + // the programmer forgot to deal with a case + Preconditions.checkState(false, + "Unknown arithmetic operation " + op_.toString() + " in: " + this.toSql()); + break; + } + + // Don't cast from decimal to decimal. The BE function can just handle this. + if (!(type_.isDecimal() && t0.isDecimal())) castChild(0, type_); + if (!(type_.isDecimal() && t1.isDecimal())) castChild(1, type_); + t0 = getChild(0).getType(); + t1 = getChild(1).getType(); + + fn_ = getBuiltinFunction(analyzer, fnName, collectChildReturnTypes(), + CompareMode.IS_IDENTICAL); + if (fn_ == null) { + Preconditions.checkState(false, String.format("No match " + + "for '%s' with operand types %s and %s", toSql(), t0, t1)); + } + Preconditions.checkState(type_.matchesType(fn_.getReturnType())); + } + + @Override + public Expr clone() { return new ArithmeticExpr(this); } +}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/AuthorizationStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AuthorizationStmt.java b/fe/src/main/java/org/apache/impala/analysis/AuthorizationStmt.java new file mode 100644 index 0000000..4e88014 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/AuthorizationStmt.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.authorization.User; +import com.cloudera.impala.common.AnalysisException; +import com.google.common.base.Strings; + +/** + * Base class for all authorization statements - CREATE/DROP/SHOW ROLE, GRANT/REVOKE + * ROLE/privilege, etc. + */ +public class AuthorizationStmt extends StatementBase { + // Set during analysis + protected User requestingUser_; + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (!analyzer.getAuthzConfig().isEnabled()) { + throw new AnalysisException("Authorization is not enabled. To enable " + + "authorization restart Impala with the --server_name=<name> flag."); + } + if (analyzer.getAuthzConfig().isFileBasedPolicy()) { + throw new AnalysisException("Cannot execute authorization statement using a file" + + " based policy. To disable file based policies, restart Impala without the " + + "-authorization_policy_file flag set."); + } + if (Strings.isNullOrEmpty(analyzer.getUser().getName())) { + throw new AnalysisException("Cannot execute authorization statement with an " + + "empty username."); + } + requestingUser_ = analyzer.getUser(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java b/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java new file mode 100644 index 0000000..69780e0 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.catalog.HdfsTable; +import com.cloudera.impala.catalog.Table; +import com.cloudera.impala.common.AnalysisException; +import com.google.common.base.Preconditions; + +/** + * Represents a reference to an actual table, such as an Hdfs or HBase table. + * BaseTableRefs are instantiated as a result of table resolution during analysis + * of a SelectStmt. + */ +public class BaseTableRef extends TableRef { + + /** + * Create a BaseTableRef from the original unresolved table ref as well as + * its resolved path. Sets table aliases and join-related attributes. + */ + public BaseTableRef(TableRef tableRef, Path resolvedPath) { + super(tableRef); + Preconditions.checkState(resolvedPath.isResolved()); + Preconditions.checkState(resolvedPath.isRootedAtTable()); + resolvedPath_ = resolvedPath; + // Set implicit aliases if no explicit one was given. + if (hasExplicitAlias()) return; + aliases_ = new String[] { + getTable().getTableName().toString().toLowerCase(), + getTable().getName().toLowerCase() }; + } + + /** + * C'tor for cloning. + */ + private BaseTableRef(BaseTableRef other) { + super(other); + } + + /** + * Register this table ref and then analyze the Join clause. + */ + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (isAnalyzed_) return; + analyzer.registerAuthAndAuditEvent(resolvedPath_.getRootTable(), analyzer); + desc_ = analyzer.registerTableRef(this); + isAnalyzed_ = true; + analyzeHints(analyzer); + analyzeJoin(analyzer); + analyzeSkipHeaderLineCount(); + } + + @Override + protected String tableRefToSql() { + // Enclose the alias in quotes if Hive cannot parse it without quotes. + // This is needed for view compatibility between Impala and Hive. + String aliasSql = null; + String alias = getExplicitAlias(); + if (alias != null) aliasSql = ToSqlUtils.getIdentSql(alias); + String tableHintsSql = ToSqlUtils.getPlanHintsSql(tableHints_); + return getTable().getTableName().toSql() + + ((aliasSql != null) ? " " + aliasSql : "") + + (tableHintsSql != "" ? " " + tableHintsSql : ""); + } + + public String debugString() { return tableRefToSql(); } + @Override + protected TableRef clone() { return new BaseTableRef(this); } + + /** + * Analyze the 'skip.header.line.count' property. + */ + private void analyzeSkipHeaderLineCount() throws AnalysisException { + Table table = getTable(); + if (!(table instanceof HdfsTable)) return; + HdfsTable hdfsTable = (HdfsTable)table; + + StringBuilder error = new StringBuilder(); + hdfsTable.parseSkipHeaderLineCount(error); + if (error.length() > 0) throw new AnalysisException(error.toString()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/BetweenPredicate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/BetweenPredicate.java b/fe/src/main/java/org/apache/impala/analysis/BetweenPredicate.java new file mode 100644 index 0000000..d76a4c6 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/BetweenPredicate.java @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import java.util.ArrayList; +import java.util.List; + +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TExprNode; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +/** + * Class describing between predicates. After successful analysis, we rewrite + * the between predicate to a conjunctive/disjunctive compound predicate + * to be handed to the backend. + */ +public class BetweenPredicate extends Predicate { + + private final boolean isNotBetween_; + + // After successful analysis, we rewrite this between predicate + // into a conjunctive/disjunctive compound predicate. + private CompoundPredicate rewrittenPredicate_; + + // Children of the BetweenPredicate, since this.children should hold the children + // of the rewritten predicate to make sure toThrift() picks up the right ones. + private ArrayList<Expr> originalChildren_ = Lists.newArrayList(); + + // First child is the comparison expr which should be in [lowerBound, upperBound]. + public BetweenPredicate(Expr compareExpr, Expr lowerBound, Expr upperBound, + boolean isNotBetween) { + originalChildren_.add(compareExpr); + originalChildren_.add(lowerBound); + originalChildren_.add(upperBound); + this.isNotBetween_ = isNotBetween; + } + + /** + * Copy c'tor used in clone(). + */ + protected BetweenPredicate(BetweenPredicate other) { + super(other); + isNotBetween_ = other.isNotBetween_; + originalChildren_ = Expr.cloneList(other.originalChildren_); + if (other.rewrittenPredicate_ != null) { + rewrittenPredicate_ = (CompoundPredicate) other.rewrittenPredicate_.clone(); + } + } + + public CompoundPredicate getRewrittenPredicate() { + Preconditions.checkState(isAnalyzed_); + return rewrittenPredicate_; + } + public ArrayList<Expr> getOriginalChildren() { return originalChildren_; } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (isAnalyzed_) return; + super.analyze(analyzer); + if (originalChildren_.get(0) instanceof Subquery && + (originalChildren_.get(1) instanceof Subquery || + originalChildren_.get(2) instanceof Subquery)) { + throw new AnalysisException("Comparison between subqueries is not " + + "supported in a between predicate: " + toSqlImpl()); + } + analyzer.castAllToCompatibleType(originalChildren_); + + // Rewrite between predicate into a conjunctive/disjunctive compound predicate. + if (isNotBetween_) { + // Rewrite into disjunction. + Predicate lower = new BinaryPredicate(BinaryPredicate.Operator.LT, + originalChildren_.get(0), originalChildren_.get(1)); + Predicate upper = new BinaryPredicate(BinaryPredicate.Operator.GT, + originalChildren_.get(0), originalChildren_.get(2)); + rewrittenPredicate_ = + new CompoundPredicate(CompoundPredicate.Operator.OR, lower, upper); + } else { + // Rewrite into conjunction. + Predicate lower = new BinaryPredicate(BinaryPredicate.Operator.GE, + originalChildren_.get(0), originalChildren_.get(1)); + Predicate upper = new BinaryPredicate(BinaryPredicate.Operator.LE, + originalChildren_.get(0), originalChildren_.get(2)); + rewrittenPredicate_ = + new CompoundPredicate(CompoundPredicate.Operator.AND, lower, upper); + } + + try { + rewrittenPredicate_.analyze(analyzer); + fn_ = rewrittenPredicate_.fn_; + } catch (AnalysisException e) { + // We should have already guaranteed that analysis will succeed. + Preconditions.checkState(false, "Analysis failed in rewritten between predicate"); + } + + // Make sure toThrift() picks up the children of the rewritten predicate. + children_ = rewrittenPredicate_.getChildren(); + // Since the only child is a CompoundPredicate expressing the comparison, + // the cost of the comparison is fully captured by the children's cost. + evalCost_ = getChildCosts(); + isAnalyzed_ = true; + } + + @Override + public List<Expr> getConjuncts() { + return rewrittenPredicate_.getConjuncts(); + } + + @Override + protected void toThrift(TExprNode msg) { + rewrittenPredicate_.toThrift(msg); + } + + @Override + public String toSqlImpl() { + String notStr = (isNotBetween_) ? "NOT " : ""; + return originalChildren_.get(0).toSql() + " " + notStr + "BETWEEN " + + originalChildren_.get(1).toSql() + " AND " + originalChildren_.get(2).toSql(); + } + + /** + * Also substitute the exprs in originalChildren when cloning. + */ + @Override + protected Expr substituteImpl(ExprSubstitutionMap smap, Analyzer analyzer) + throws AnalysisException { + BetweenPredicate clone = (BetweenPredicate) super.substituteImpl(smap, analyzer); + Preconditions.checkNotNull(clone); + clone.originalChildren_ = + Expr.substituteList(originalChildren_, smap, analyzer, false); + return clone; + } + + @Override + public Expr clone() { return new BetweenPredicate(this); } + + @Override + public Expr reset() { + super.reset(); + originalChildren_ = Expr.resetList(originalChildren_); + return this; + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java b/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java new file mode 100644 index 0000000..35d03e1 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java @@ -0,0 +1,388 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cloudera.impala.catalog.Db; +import com.cloudera.impala.catalog.Function.CompareMode; +import com.cloudera.impala.catalog.ScalarFunction; +import com.cloudera.impala.catalog.Type; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.common.Pair; +import com.cloudera.impala.common.Reference; +import com.cloudera.impala.extdatasource.thrift.TComparisonOp; +import com.cloudera.impala.thrift.TExprNode; +import com.cloudera.impala.thrift.TExprNodeType; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.Lists; + +/** + * Most predicates with two operands. + * + */ +public class BinaryPredicate extends Predicate { + private final static Logger LOG = LoggerFactory.getLogger(BinaryPredicate.class); + + // true if this BinaryPredicate is inferred from slot equivalences, false otherwise. + private boolean isInferred_ = false; + + public enum Operator { + EQ("=", "eq", TComparisonOp.EQ), + NE("!=", "ne", TComparisonOp.NE), + LE("<=", "le", TComparisonOp.LE), + GE(">=", "ge", TComparisonOp.GE), + LT("<", "lt", TComparisonOp.LT), + GT(">", "gt", TComparisonOp.GT), + DISTINCT_FROM("IS DISTINCT FROM", "distinctfrom", TComparisonOp.DISTINCT_FROM), + NOT_DISTINCT("IS NOT DISTINCT FROM", "notdistinct", TComparisonOp.NOT_DISTINCT), + // Same as EQ, except it returns True if the rhs is NULL. There is no backend + // function for this. The functionality is embedded in the hash-join + // implementation. + NULL_MATCHING_EQ("=", "null_matching_eq", TComparisonOp.EQ); + + private final String description_; + private final String name_; + private final TComparisonOp thriftOp_; + + private Operator(String description, String name, TComparisonOp thriftOp) { + this.description_ = description; + this.name_ = name; + this.thriftOp_ = thriftOp; + } + + @Override + public String toString() { return description_; } + public String getName() { return name_; } + public TComparisonOp getThriftOp() { return thriftOp_; } + public boolean isEquivalence() { return this == EQ || this == NOT_DISTINCT; } + + public Operator converse() { + switch (this) { + case EQ: return EQ; + case NE: return NE; + case LE: return GE; + case GE: return LE; + case LT: return GT; + case GT: return LT; + case DISTINCT_FROM: return DISTINCT_FROM; + case NOT_DISTINCT: return NOT_DISTINCT; + case NULL_MATCHING_EQ: + throw new IllegalStateException("Not implemented"); + default: throw new IllegalStateException("Invalid operator"); + } + } + } + + public static void initBuiltins(Db db) { + for (Type t: Type.getSupportedTypes()) { + if (t.isNull()) continue; // NULL is handled through type promotion. + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.EQ.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.NE.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.LE.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.GE.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.LT.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); + db.addBuiltin(ScalarFunction.createBuiltinOperator( + Operator.GT.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); + } + } + + /** + * Normalizes a 'predicate' consisting of an uncast SlotRef and a constant Expr into + * the following form: <SlotRef> <Op> <LiteralExpr> + * If 'predicate' cannot be expressed in this way, null is returned. + */ + public static BinaryPredicate normalizeSlotRefComparison(BinaryPredicate predicate, + Analyzer analyzer) { + SlotRef ref = null; + if (predicate.getChild(0) instanceof SlotRef) { + ref = (SlotRef) predicate.getChild(0); + } else if (predicate.getChild(1) instanceof SlotRef) { + ref = (SlotRef) predicate.getChild(1); + } + + if (ref == null) return null; + if (ref != predicate.getChild(0)) { + Preconditions.checkState(ref == predicate.getChild(1)); + predicate = new BinaryPredicate(predicate.getOp().converse(), ref, + predicate.getChild(0)); + predicate.analyzeNoThrow(analyzer); + } + + try { + predicate.foldConstantChildren(analyzer); + } catch (AnalysisException ex) { + // Throws if the expression cannot be evaluated by the BE. + return null; + } + predicate.analyzeNoThrow(analyzer); + if (!(predicate.getChild(1) instanceof LiteralExpr)) return null; + return predicate; + } + + private Operator op_; + + public Operator getOp() { return op_; } + public void setOp(Operator op) { op_ = op; } + + public BinaryPredicate(Operator op, Expr e1, Expr e2) { + super(); + this.op_ = op; + Preconditions.checkNotNull(e1); + children_.add(e1); + Preconditions.checkNotNull(e2); + children_.add(e2); + } + + protected BinaryPredicate(BinaryPredicate other) { + super(other); + op_ = other.op_; + isInferred_ = other.isInferred_; + } + + public boolean isNullMatchingEq() { return op_ == Operator.NULL_MATCHING_EQ; } + + public boolean isInferred() { return isInferred_; } + public void setIsInferred() { isInferred_ = true; } + + @Override + public String toSqlImpl() { + return getChild(0).toSql() + " " + op_.toString() + " " + getChild(1).toSql(); + } + + @Override + protected void toThrift(TExprNode msg) { + Preconditions.checkState(children_.size() == 2); + // Cannot serialize a nested predicate. + Preconditions.checkState(!contains(Subquery.class)); + // This check is important because we often clone and/or evaluate predicates, + // and it's easy to get the casting logic wrong, e.g., cloned predicates + // with expr substitutions need to be re-analyzed with reanalyze(). + Preconditions.checkState(getChild(0).getType().getPrimitiveType() == + getChild(1).getType().getPrimitiveType(), + "child 0 type: " + getChild(0).getType() + + " child 1 type: " + getChild(1).getType()); + msg.node_type = TExprNodeType.FUNCTION_CALL; + } + + @Override + public String debugString() { + return Objects.toStringHelper(this) + .add("op", op_) + .addValue(super.debugString()) + .toString(); + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (isAnalyzed_) return; + super.analyze(analyzer); + + convertNumericLiteralsFromDecimal(analyzer); + String opName = op_.getName().equals("null_matching_eq") ? "eq" : op_.getName(); + fn_ = getBuiltinFunction(analyzer, opName, collectChildReturnTypes(), + CompareMode.IS_NONSTRICT_SUPERTYPE_OF); + if (fn_ == null) { + // Construct an appropriate error message and throw an AnalysisException. + String errMsg = "operands of type " + getChild(0).getType().toSql() + " and " + + getChild(1).getType().toSql() + " are not comparable: " + toSql(); + + // Check if any of the children is a Subquery that does not return a + // scalar. + for (Expr expr: children_) { + if (expr instanceof Subquery && !expr.getType().isScalarType()) { + errMsg = "Subquery must return a single row: " + expr.toSql(); + break; + } + } + + throw new AnalysisException(errMsg); + } + Preconditions.checkState(fn_.getReturnType().isBoolean()); + + ArrayList<Expr> subqueries = Lists.newArrayList(); + collectAll(Predicates.instanceOf(Subquery.class), subqueries); + if (subqueries.size() > 1) { + // TODO Remove that restriction when we add support for independent subquery + // evaluation. + throw new AnalysisException("Multiple subqueries are not supported in binary " + + "predicates: " + toSql()); + } + if (contains(ExistsPredicate.class)) { + throw new AnalysisException("EXISTS subquery predicates are not " + + "supported in binary predicates: " + toSql()); + } + + List<InPredicate> inPredicates = Lists.newArrayList(); + collect(InPredicate.class, inPredicates); + for (InPredicate inPredicate: inPredicates) { + if (inPredicate.contains(Subquery.class)) { + throw new AnalysisException("IN subquery predicates are not supported in " + + "binary predicates: " + toSql()); + } + } + + // Don't perform any casting for predicates with subqueries here. Any casting + // required will be performed when the subquery is unnested. + if (!contains(Subquery.class)) castForFunctionCall(true); + + // Determine selectivity + // TODO: Compute selectivity for nested predicates. + // TODO: Improve estimation using histograms. + Reference<SlotRef> slotRefRef = new Reference<SlotRef>(); + if ((op_ == Operator.EQ || op_ == Operator.NOT_DISTINCT) + && isSingleColumnPredicate(slotRefRef, null)) { + long distinctValues = slotRefRef.getRef().getNumDistinctValues(); + if (distinctValues > 0) { + selectivity_ = 1.0 / distinctValues; + selectivity_ = Math.max(0, Math.min(1, selectivity_)); + } + } + + // Compute cost. + if (hasChildCosts()) { + if (getChild(0).getType().isFixedLengthType()) { + evalCost_ = getChildCosts() + BINARY_PREDICATE_COST; + } else if (getChild(0).getType().isStringType()) { + evalCost_ = getChildCosts() + + (float) (getAvgStringLength(getChild(0)) + getAvgStringLength(getChild(1)) * + BINARY_PREDICATE_COST); + } else { + //TODO(tmarshall): Handle other var length types here. + evalCost_ = getChildCosts() + VAR_LEN_BINARY_PREDICATE_COST; + } + } + } + + /** + * If predicate is of the form "<slotref> <op> <expr>", returns expr, + * otherwise returns null. Slotref may be wrapped in a CastExpr. + * TODO: revisit CAST handling at the caller + */ + public Expr getSlotBinding(SlotId id) { + // check left operand + SlotRef slotRef = getChild(0).unwrapSlotRef(false); + if (slotRef != null && slotRef.getSlotId() == id) return getChild(1); + // check right operand + slotRef = getChild(1).unwrapSlotRef(false); + if (slotRef != null && slotRef.getSlotId() == id) return getChild(0); + return null; + } + + /** + * If e is an equality predicate between two slots that only require implicit + * casts, returns those two slots; otherwise returns null. + */ + public static Pair<SlotId, SlotId> getEqSlots(Expr e) { + if (!(e instanceof BinaryPredicate)) return null; + return ((BinaryPredicate) e).getEqSlots(); + } + + /** + * If this is an equality predicate between two slots that only require implicit + * casts, returns those two slots; otherwise returns null. + */ + @Override + public Pair<SlotId, SlotId> getEqSlots() { + if (op_ != Operator.EQ) return null; + SlotRef lhs = getChild(0).unwrapSlotRef(true); + if (lhs == null) return null; + SlotRef rhs = getChild(1).unwrapSlotRef(true); + if (rhs == null) return null; + return new Pair<SlotId, SlotId>(lhs.getSlotId(), rhs.getSlotId()); + } + + /** + * If predicate is of the form "<SlotRef> op <Expr>" or "<Expr> op <SlotRef>", + * returns the SlotRef, otherwise returns null. + */ + @Override + public SlotRef getBoundSlot() { + SlotRef slotRef = getChild(0).unwrapSlotRef(true); + if (slotRef != null) return slotRef; + return getChild(1).unwrapSlotRef(true); + } + + /** + * Negates a BinaryPredicate. + */ + @Override + public Expr negate() { + Operator newOp = null; + switch (op_) { + case EQ: + newOp = Operator.NE; + break; + case NE: + newOp = Operator.EQ; + break; + case LT: + newOp = Operator.GE; + break; + case LE: + newOp = Operator.GT; + break; + case GE: + newOp = Operator.LT; + break; + case GT: + newOp = Operator.LE; + break; + case DISTINCT_FROM: + newOp = Operator.NOT_DISTINCT; + break; + case NOT_DISTINCT: + newOp = Operator.DISTINCT_FROM; + break; + case NULL_MATCHING_EQ: + throw new IllegalStateException("Not implemented"); + } + return new BinaryPredicate(newOp, getChild(0), getChild(1)); + } + + /** + * Swaps the first with the second child in-place. Only valid to call for + * equivalence and not equal predicates. + */ + public void reverse() { + Preconditions.checkState(op_.isEquivalence() || op_ == Operator.NE); + Collections.swap(children_, 0, 1); + } + + @Override + public boolean equals(Object obj) { + if (!super.equals(obj)) return false; + BinaryPredicate other = (BinaryPredicate) obj; + return op_.equals(other.op_); + } + + @Override + public Expr clone() { return new BinaryPredicate(this); } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/BoolLiteral.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/BoolLiteral.java b/fe/src/main/java/org/apache/impala/analysis/BoolLiteral.java new file mode 100644 index 0000000..03b2b1f --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/BoolLiteral.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.catalog.Type; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TBoolLiteral; +import com.cloudera.impala.thrift.TExprNode; +import com.cloudera.impala.thrift.TExprNodeType; +import com.google.common.base.Objects; + +public class BoolLiteral extends LiteralExpr { + private final boolean value_; + + public BoolLiteral(boolean value) { + this.value_ = value; + type_ = Type.BOOLEAN; + evalCost_ = LITERAL_COST; + } + + public BoolLiteral(String value) throws AnalysisException { + type_ = Type.BOOLEAN; + evalCost_ = LITERAL_COST; + if (value.toLowerCase().equals("true")) { + this.value_ = true; + } else if (value.toLowerCase().equals("false")) { + this.value_ = false; + } else { + throw new AnalysisException("invalid BOOLEAN literal: " + value); + } + } + + /** + * Copy c'tor used in clone. + */ + protected BoolLiteral(BoolLiteral other) { + super(other); + value_ = other.value_; + } + + @Override + public String debugString() { + return Objects.toStringHelper(this) + .add("value", value_) + .toString(); + } + + @Override + public boolean equals(Object obj) { + if (!super.equals(obj)) { + return false; + } + return ((BoolLiteral) obj).value_ == value_; + } + + @Override + public int hashCode() { return value_ ? 1 : 0; } + + public boolean getValue() { return value_; } + + @Override + public String toSqlImpl() { + return getStringValue(); + } + + @Override + public String getStringValue() { + return value_ ? "TRUE" : "FALSE"; + } + + @Override + protected void toThrift(TExprNode msg) { + msg.node_type = TExprNodeType.BOOL_LITERAL; + msg.bool_literal = new TBoolLiteral(value_); + } + + @Override + protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { + if (targetType.equals(this.type_)) { + return this; + } else { + return new CastExpr(targetType, this); + } + } + + @Override + public int compareTo(LiteralExpr o) { + int ret = super.compareTo(o); + if (ret != 0) return ret; + BoolLiteral other = (BoolLiteral) o; + if (value_ && !other.getValue()) return 1; + if (!value_ && other.getValue()) return -1; + return 0; + } + + @Override + public Expr clone() { return new BoolLiteral(this); } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/CaseExpr.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/CaseExpr.java b/fe/src/main/java/org/apache/impala/analysis/CaseExpr.java new file mode 100644 index 0000000..bd3ec83 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/CaseExpr.java @@ -0,0 +1,379 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import java.util.List; + +import com.cloudera.impala.catalog.Db; +import com.cloudera.impala.catalog.Function.CompareMode; +import com.cloudera.impala.catalog.PrimitiveType; +import com.cloudera.impala.catalog.ScalarFunction; +import com.cloudera.impala.catalog.ScalarType; +import com.cloudera.impala.catalog.Type; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TCaseExpr; +import com.cloudera.impala.thrift.TExprNode; +import com.cloudera.impala.thrift.TExprNodeType; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +/** + * CASE and DECODE are represented using this class. The backend implementation is + * always the "case" function. + * + * The internal representation of + * CASE [expr] WHEN expr THEN expr [WHEN expr THEN expr ...] [ELSE expr] END + * Each When/Then is stored as two consecutive children (whenExpr, thenExpr). If a case + * expr is given then it is the first child. If an else expr is given then it is the + * last child. + * + * The internal representation of + * DECODE(expr, key_expr, val_expr [, key_expr, val_expr ...] [, default_val_expr]) + * has a pair of children for each pair of key/val_expr and an additional child if the + * default_val_expr was given. The first child represents the comparison of expr to + * key_expr. Decode has three forms: + * 1) DECODE(expr, null_literal, val_expr) - + * child[0] = IsNull(expr) + * 2) DECODE(expr, non_null_literal, val_expr) - + * child[0] = Eq(expr, literal) + * 3) DECODE(expr1, expr2, val_expr) - + * child[0] = Or(And(IsNull(expr1), IsNull(expr2)), Eq(expr1, expr2)) + * The children representing val_expr (child[1]) and default_val_expr (child[2]) are + * simply the exprs themselves. + * + * Example of equivalent CASE for DECODE(foo, 'bar', 1, col, 2, NULL, 3, 4): + * CASE + * WHEN foo = 'bar' THEN 1 -- no need for IS NULL check + * WHEN foo IS NULL AND col IS NULL OR foo = col THEN 2 + * WHEN foo IS NULL THEN 3 -- no need for equality check + * ELSE 4 + * END + */ +public class CaseExpr extends Expr { + + // Set if constructed from a DECODE, null otherwise. + private FunctionCallExpr decodeExpr_; + + private boolean hasCaseExpr_; + private boolean hasElseExpr_; + + public CaseExpr(Expr caseExpr, List<CaseWhenClause> whenClauses, Expr elseExpr) { + super(); + if (caseExpr != null) { + children_.add(caseExpr); + hasCaseExpr_ = true; + } + for (CaseWhenClause whenClause: whenClauses) { + Preconditions.checkNotNull(whenClause.getWhenExpr()); + children_.add(whenClause.getWhenExpr()); + Preconditions.checkNotNull(whenClause.getThenExpr()); + children_.add(whenClause.getThenExpr()); + } + if (elseExpr != null) { + children_.add(elseExpr); + hasElseExpr_ = true; + } + } + + /** + * Constructs an equivalent CaseExpr representation. + * + * The DECODE behavior is basically the same as the hasCaseExpr_ version of CASE. + * Though there is one difference. NULLs are considered equal when comparing the + * argument to be decoded with the candidates. This differences is for compatibility + * with Oracle. http://docs.oracle.com/cd/B19306_01/server.102/b14200/functions040.htm. + * To account for the difference, the CASE representation will use the non-hasCaseExpr_ + * version. + * + * The return type of DECODE differs from that of Oracle when the third argument is + * the NULL literal. In Oracle the return type is STRING. In Impala the return type is + * determined by the implicit casting rules (i.e. it's not necessarily a STRING). This + * is done so seemingly normal usages such as DECODE(int_col, tinyint_col, NULL, + * bigint_col) will avoid type check errors (STRING incompatible with BIGINT). + */ + public CaseExpr(FunctionCallExpr decodeExpr) { + super(); + decodeExpr_ = decodeExpr; + hasCaseExpr_ = false; + + int childIdx = 0; + Expr encoded = null; + Expr encodedIsNull = null; + if (!decodeExpr.getChildren().isEmpty()) { + encoded = decodeExpr.getChild(childIdx++); + encodedIsNull = new IsNullPredicate(encoded, false); + } + + // Add the key_expr/val_expr pairs + while (childIdx + 2 <= decodeExpr.getChildren().size()) { + Expr candidate = decodeExpr.getChild(childIdx++); + if (candidate.isLiteral()) { + if (candidate.isNullLiteral()) { + // An example case is DECODE(foo, NULL, bar), since NULLs are considered + // equal, this becomes CASE WHEN foo IS NULL THEN bar END. + children_.add(encodedIsNull); + } else { + children_.add(new BinaryPredicate( + BinaryPredicate.Operator.EQ, encoded, candidate)); + } + } else { + children_.add(new CompoundPredicate(CompoundPredicate.Operator.OR, + new CompoundPredicate(CompoundPredicate.Operator.AND, + encodedIsNull, new IsNullPredicate(candidate, false)), + new BinaryPredicate(BinaryPredicate.Operator.EQ, encoded, candidate))); + } + + // Add the value + children_.add(decodeExpr.getChild(childIdx++)); + } + + // Add the default value + if (childIdx < decodeExpr.getChildren().size()) { + hasElseExpr_ = true; + children_.add(decodeExpr.getChild(childIdx)); + } + } + + /** + * Copy c'tor used in clone(). + */ + protected CaseExpr(CaseExpr other) { + super(other); + decodeExpr_ = other.decodeExpr_; + hasCaseExpr_ = other.hasCaseExpr_; + hasElseExpr_ = other.hasElseExpr_; + } + + public static void initBuiltins(Db db) { + for (Type t: Type.getSupportedTypes()) { + if (t.isNull()) continue; + if (t.isScalarType(PrimitiveType.CHAR)) continue; + // TODO: case is special and the signature cannot be represented. + // It is alternating varargs + // e.g. case(bool, type, bool type, bool type, etc). + // Instead we just add a version for each of the return types + // e.g. case(BOOLEAN), case(INT), etc + db.addBuiltin(ScalarFunction.createBuiltinOperator( + "case", "", Lists.newArrayList(t), t)); + // Same for DECODE + db.addBuiltin(ScalarFunction.createBuiltinOperator( + "decode", "", Lists.newArrayList(t), t)); + } + } + + @Override + public boolean equals(Object obj) { + if (!super.equals(obj)) return false; + CaseExpr expr = (CaseExpr) obj; + return hasCaseExpr_ == expr.hasCaseExpr_ + && hasElseExpr_ == expr.hasElseExpr_ + && isDecode() == expr.isDecode(); + } + + @Override + public String toSqlImpl() { + return (decodeExpr_ == null) ? toCaseSql() : decodeExpr_.toSqlImpl(); + } + + @VisibleForTesting + String toCaseSql() { + StringBuilder output = new StringBuilder("CASE"); + int childIdx = 0; + if (hasCaseExpr_) { + output.append(" " + children_.get(childIdx++).toSql()); + } + while (childIdx + 2 <= children_.size()) { + output.append(" WHEN " + children_.get(childIdx++).toSql()); + output.append(" THEN " + children_.get(childIdx++).toSql()); + } + if (hasElseExpr_) { + output.append(" ELSE " + children_.get(children_.size() - 1).toSql()); + } + output.append(" END"); + return output.toString(); + } + + @Override + protected void toThrift(TExprNode msg) { + msg.node_type = TExprNodeType.CASE_EXPR; + msg.case_expr = new TCaseExpr(hasCaseExpr_, hasElseExpr_); + } + + private void castCharToString(int childIndex) throws AnalysisException { + if (children_.get(childIndex).getType().isScalarType(PrimitiveType.CHAR)) { + children_.set(childIndex, children_.get(childIndex).castTo(ScalarType.STRING)); + } + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (isAnalyzed_) return; + super.analyze(analyzer); + + if (isDecode()) { + Preconditions.checkState(!hasCaseExpr_); + // decodeExpr_.analyze() would fail validating function existence. The complex + // vararg signature is currently unsupported. + FunctionCallExpr.validateScalarFnParams(decodeExpr_.getParams()); + if (decodeExpr_.getChildren().size() < 3) { + throw new AnalysisException("DECODE in '" + toSql() + "' requires at least 3 " + + "arguments."); + } + } + + // Since we have no BE implementation of a CaseExpr with CHAR types, + // we cast the CHAR-typed whenExprs and caseExprs to STRING, + // TODO: This casting is not always correct and needs to be fixed, see IMPALA-1652. + + // Keep track of maximum compatible type of case expr and all when exprs. + Type whenType = null; + // Keep track of maximum compatible type of else expr and all then exprs. + Type returnType = null; + // Remember last of these exprs for error reporting. + Expr lastCompatibleThenExpr = null; + Expr lastCompatibleWhenExpr = null; + int loopEnd = children_.size(); + if (hasElseExpr_) { + --loopEnd; + } + int loopStart; + Expr caseExpr = null; + // Set loop start, and initialize returnType as type of castExpr. + if (hasCaseExpr_) { + loopStart = 1; + castCharToString(0); + caseExpr = children_.get(0); + caseExpr.analyze(analyzer); + whenType = caseExpr.getType(); + lastCompatibleWhenExpr = children_.get(0); + } else { + whenType = Type.BOOLEAN; + loopStart = 0; + } + + // Go through when/then exprs and determine compatible types. + for (int i = loopStart; i < loopEnd; i += 2) { + castCharToString(i); + Expr whenExpr = children_.get(i); + if (hasCaseExpr_) { + // Determine maximum compatible type of the case expr, + // and all when exprs seen so far. We will add casts to them at the very end. + whenType = analyzer.getCompatibleType(whenType, + lastCompatibleWhenExpr, whenExpr); + lastCompatibleWhenExpr = whenExpr; + } else { + // If no case expr was given, then the when exprs should always return + // boolean or be castable to boolean. + if (!Type.isImplicitlyCastable(whenExpr.getType(), Type.BOOLEAN, false)) { + Preconditions.checkState(isCase()); + throw new AnalysisException("When expr '" + whenExpr.toSql() + "'" + + " is not of type boolean and not castable to type boolean."); + } + // Add a cast if necessary. + if (!whenExpr.getType().isBoolean()) castChild(Type.BOOLEAN, i); + } + // Determine maximum compatible type of the then exprs seen so far. + // We will add casts to them at the very end. + Expr thenExpr = children_.get(i + 1); + returnType = analyzer.getCompatibleType(returnType, + lastCompatibleThenExpr, thenExpr); + lastCompatibleThenExpr = thenExpr; + } + if (hasElseExpr_) { + Expr elseExpr = children_.get(children_.size() - 1); + returnType = analyzer.getCompatibleType(returnType, + lastCompatibleThenExpr, elseExpr); + } + + // Make sure BE doesn't see TYPE_NULL by picking an arbitrary type + if (whenType.isNull()) whenType = ScalarType.BOOLEAN; + if (returnType.isNull()) returnType = ScalarType.BOOLEAN; + + // Add casts to case expr to compatible type. + if (hasCaseExpr_) { + // Cast case expr. + if (!children_.get(0).type_.equals(whenType)) { + castChild(whenType, 0); + } + // Add casts to when exprs to compatible type. + for (int i = loopStart; i < loopEnd; i += 2) { + if (!children_.get(i).type_.equals(whenType)) { + castChild(whenType, i); + } + } + } + // Cast then exprs to compatible type. + for (int i = loopStart + 1; i < children_.size(); i += 2) { + if (!children_.get(i).type_.equals(returnType)) { + castChild(returnType, i); + } + } + // Cast else expr to compatible type. + if (hasElseExpr_) { + if (!children_.get(children_.size() - 1).type_.equals(returnType)) { + castChild(returnType, children_.size() - 1); + } + } + + // Do the function lookup just based on the whenType. + Type[] args = new Type[1]; + args[0] = whenType; + fn_ = getBuiltinFunction(analyzer, "case", args, + CompareMode.IS_NONSTRICT_SUPERTYPE_OF); + Preconditions.checkNotNull(fn_); + type_ = returnType; + + // Compute cost as the sum of evaluating all of the WHEN exprs, plus + // the max of the THEN/ELSE exprs. + float maxThenCost = 0; + float whenCosts = 0; + boolean hasChildCosts = true; + for (int i = 0; i < children_.size(); ++i) { + if (!getChild(i).hasCost()) { + hasChildCosts = false; + break; + } + + if (hasCaseExpr_ && i % 2 == 1) { + // This child is a WHEN expr. BINARY_PREDICATE_COST accounts for the cost of + // comparing the CASE expr to the WHEN expr. + whenCosts += getChild(0).getCost() + getChild(i).getCost() + + BINARY_PREDICATE_COST; + } else if (!hasCaseExpr_ && i % 2 == 0) { + // This child is a WHEN expr. + whenCosts += getChild(i).getCost(); + } else if (i != 0) { + // This child is a THEN or ELSE expr. + float thenCost = getChild(i).getCost(); + if (thenCost > maxThenCost) maxThenCost = thenCost; + } + } + if (hasChildCosts) { + evalCost_ = whenCosts + maxThenCost; + } + } + + private boolean isCase() { return !isDecode(); } + private boolean isDecode() { return decodeExpr_ != null; } + public boolean hasCaseExpr() { return hasCaseExpr_; } + public boolean hasElseExpr() { return hasElseExpr_; } + + @Override + public Expr clone() { return new CaseExpr(this); } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/CaseWhenClause.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/CaseWhenClause.java b/fe/src/main/java/org/apache/impala/analysis/CaseWhenClause.java new file mode 100644 index 0000000..8b1433e --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/CaseWhenClause.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + + +/** + * captures info of a single WHEN expr THEN expr clause. + * + */ +class CaseWhenClause { + private final Expr whenExpr_; + private final Expr thenExpr_; + + public CaseWhenClause(Expr whenExpr, Expr thenExpr) { + super(); + this.whenExpr_ = whenExpr; + this.thenExpr_ = thenExpr; + } + + public Expr getWhenExpr() { + return whenExpr_; + } + + public Expr getThenExpr() { + return thenExpr_; + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/CastExpr.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/CastExpr.java b/fe/src/main/java/org/apache/impala/analysis/CastExpr.java new file mode 100644 index 0000000..2b3b271 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/CastExpr.java @@ -0,0 +1,312 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.catalog.Catalog; +import com.cloudera.impala.catalog.Db; +import com.cloudera.impala.catalog.Function; +import com.cloudera.impala.catalog.Function.CompareMode; +import com.cloudera.impala.catalog.PrimitiveType; +import com.cloudera.impala.catalog.ScalarFunction; +import com.cloudera.impala.catalog.ScalarType; +import com.cloudera.impala.catalog.Type; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TExpr; +import com.cloudera.impala.thrift.TExprNode; +import com.cloudera.impala.thrift.TExprNodeType; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +public class CastExpr extends Expr { + // Only set for explicit casts. Null for implicit casts. + private final TypeDef targetTypeDef_; + + // True if this is a "pre-analyzed" implicit cast. + private final boolean isImplicit_; + + // True if this cast does not change the type. + private boolean noOp_ = false; + + /** + * C'tor for "pre-analyzed" implicit casts. + */ + public CastExpr(Type targetType, Expr e) { + super(); + Preconditions.checkState(targetType.isValid()); + Preconditions.checkNotNull(e); + type_ = targetType; + targetTypeDef_ = null; + isImplicit_ = true; + // replace existing implicit casts + if (e instanceof CastExpr) { + CastExpr castExpr = (CastExpr) e; + if (castExpr.isImplicit()) e = castExpr.getChild(0); + } + children_.add(e); + + // Implicit casts don't call analyze() + // TODO: this doesn't seem like the cleanest approach but there are places + // we generate these (e.g. table loading) where there is no analyzer object. + try { + analyze(); + computeNumDistinctValues(); + } catch (AnalysisException ex) { + Preconditions.checkState(false, + "Implicit casts should never throw analysis exception."); + } + isAnalyzed_ = true; + } + + /** + * C'tor for explicit casts. + */ + public CastExpr(TypeDef targetTypeDef, Expr e) { + Preconditions.checkNotNull(targetTypeDef); + Preconditions.checkNotNull(e); + isImplicit_ = false; + targetTypeDef_ = targetTypeDef; + children_.add(e); + } + + /** + * Copy c'tor used in clone(). + */ + protected CastExpr(CastExpr other) { + super(other); + targetTypeDef_ = other.targetTypeDef_; + isImplicit_ = other.isImplicit_; + noOp_ = other.noOp_; + } + + private static String getFnName(Type targetType) { + return "castTo" + targetType.getPrimitiveType().toString(); + } + + public static void initBuiltins(Db db) { + for (Type fromType : Type.getSupportedTypes()) { + if (fromType.isNull()) continue; + for (Type toType : Type.getSupportedTypes()) { + if (toType.isNull()) continue; + // Disable casting from string to boolean + if (fromType.isStringType() && toType.isBoolean()) continue; + // Disable casting from boolean/timestamp to decimal + if ((fromType.isBoolean() || fromType.isDateType()) && toType.isDecimal()) { + continue; + } + if (fromType.getPrimitiveType() == PrimitiveType.STRING + && toType.getPrimitiveType() == PrimitiveType.CHAR) { + // Allow casting from String to Char(N) + String beSymbol = "impala::CastFunctions::CastToChar"; + db.addBuiltin(ScalarFunction.createBuiltin(getFnName(ScalarType.CHAR), + Lists.newArrayList((Type) ScalarType.STRING), false, ScalarType.CHAR, + beSymbol, null, null, true)); + continue; + } + if (fromType.getPrimitiveType() == PrimitiveType.CHAR + && toType.getPrimitiveType() == PrimitiveType.CHAR) { + // Allow casting from CHAR(N) to Char(N) + String beSymbol = "impala::CastFunctions::CastToChar"; + db.addBuiltin(ScalarFunction.createBuiltin(getFnName(ScalarType.CHAR), + Lists.newArrayList((Type) ScalarType.createCharType(-1)), false, + ScalarType.CHAR, beSymbol, null, null, true)); + continue; + } + if (fromType.getPrimitiveType() == PrimitiveType.VARCHAR + && toType.getPrimitiveType() == PrimitiveType.VARCHAR) { + // Allow casting from VARCHAR(N) to VARCHAR(M) + String beSymbol = "impala::CastFunctions::CastToStringVal"; + db.addBuiltin(ScalarFunction.createBuiltin(getFnName(ScalarType.VARCHAR), + Lists.newArrayList((Type) ScalarType.VARCHAR), false, ScalarType.VARCHAR, + beSymbol, null, null, true)); + continue; + } + if (fromType.getPrimitiveType() == PrimitiveType.VARCHAR + && toType.getPrimitiveType() == PrimitiveType.CHAR) { + // Allow casting from VARCHAR(N) to CHAR(M) + String beSymbol = "impala::CastFunctions::CastToChar"; + db.addBuiltin(ScalarFunction.createBuiltin(getFnName(ScalarType.CHAR), + Lists.newArrayList((Type) ScalarType.VARCHAR), false, ScalarType.CHAR, + beSymbol, null, null, true)); + continue; + } + if (fromType.getPrimitiveType() == PrimitiveType.CHAR + && toType.getPrimitiveType() == PrimitiveType.VARCHAR) { + // Allow casting from CHAR(N) to VARCHAR(M) + String beSymbol = "impala::CastFunctions::CastToStringVal"; + db.addBuiltin(ScalarFunction.createBuiltin(getFnName(ScalarType.VARCHAR), + Lists.newArrayList((Type) ScalarType.CHAR), false, ScalarType.VARCHAR, + beSymbol, null, null, true)); + continue; + } + // Disable no-op casts + if (fromType.equals(toType) && !fromType.isDecimal()) continue; + String beClass = toType.isDecimal() || fromType.isDecimal() ? + "DecimalOperators" : "CastFunctions"; + String beSymbol = "impala::" + beClass + "::CastTo" + Function.getUdfType(toType); + db.addBuiltin(ScalarFunction.createBuiltin(getFnName(toType), + Lists.newArrayList(fromType), false, toType, beSymbol, + null, null, true)); + } + } + } + + @Override + public String toSqlImpl() { + if (isImplicit_) return getChild(0).toSql(); + return "CAST(" + getChild(0).toSql() + " AS " + targetTypeDef_.toString() + ")"; + } + + @Override + protected void treeToThriftHelper(TExpr container) { + if (noOp_) { + getChild(0).treeToThriftHelper(container); + return; + } + super.treeToThriftHelper(container); + } + + @Override + protected void toThrift(TExprNode msg) { + msg.node_type = TExprNodeType.FUNCTION_CALL; + } + + @Override + public String debugString() { + return Objects.toStringHelper(this) + .add("isImplicit", isImplicit_) + .add("target", type_) + .addValue(super.debugString()) + .toString(); + } + + public boolean isImplicit() { return isImplicit_; } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (isAnalyzed_) return; + Preconditions.checkState(!isImplicit_); + super.analyze(analyzer); + targetTypeDef_.analyze(analyzer); + type_ = targetTypeDef_.getType(); + analyze(); + } + + private void analyze() throws AnalysisException { + if (getChild(0).hasCost()) evalCost_ = getChild(0).getCost() + CAST_COST; + + Preconditions.checkNotNull(type_); + if (type_.isComplexType()) { + throw new AnalysisException( + "Unsupported cast to complex type: " + type_.toSql()); + } + + boolean readyForCharCast = + children_.get(0).getType().getPrimitiveType() == PrimitiveType.STRING || + children_.get(0).getType().getPrimitiveType() == PrimitiveType.CHAR; + if (type_.getPrimitiveType() == PrimitiveType.CHAR && !readyForCharCast) { + // Back end functions only exist to cast string types to CHAR, there is not a cast + // for every type since it is redundant with STRING. Casts to go through 2 casts: + // (1) cast to string, to stringify the value + // (2) cast to CHAR, to truncate or pad with spaces + CastExpr tostring = new CastExpr(ScalarType.STRING, children_.get(0)); + tostring.analyze(); + children_.set(0, tostring); + } + + if (children_.get(0) instanceof NumericLiteral && type_.isFloatingPointType()) { + // Special case casting a decimal literal to a floating point number. The + // decimal literal can be interpreted as either and we want to avoid casts + // since that can result in loss of accuracy. + ((NumericLiteral)children_.get(0)).explicitlyCastToFloat(type_); + } + + if (children_.get(0).getType().isNull()) { + // Make sure BE never sees TYPE_NULL + uncheckedCastChild(type_, 0); + } + + // Ensure child has non-null type (even if it's a null literal). This is required + // for the UDF interface. + if (children_.get(0) instanceof NullLiteral) { + NullLiteral nullChild = (NullLiteral)(children_.get(0)); + nullChild.uncheckedCastTo(type_); + } + + Type childType = children_.get(0).type_; + Preconditions.checkState(!childType.isNull()); + if (childType.equals(type_)) { + noOp_ = true; + return; + } + + FunctionName fnName = new FunctionName(Catalog.BUILTINS_DB, getFnName(type_)); + Type[] args = { childType }; + Function searchDesc = new Function(fnName, args, Type.INVALID, false); + if (isImplicit_) { + fn_ = Catalog.getBuiltin(searchDesc, CompareMode.IS_NONSTRICT_SUPERTYPE_OF); + Preconditions.checkState(fn_ != null); + } else { + fn_ = Catalog.getBuiltin(searchDesc, CompareMode.IS_IDENTICAL); + if (fn_ == null) { + // allow for promotion from CHAR to STRING; only if no exact match is found + fn_ = Catalog.getBuiltin(searchDesc.promoteCharsToStrings(), + CompareMode.IS_IDENTICAL); + } + } + if (fn_ == null) { + throw new AnalysisException("Invalid type cast of " + getChild(0).toSql() + + " from " + childType + " to " + type_); + } + + Preconditions.checkState(type_.matchesType(fn_.getReturnType()), + type_ + " != " + fn_.getReturnType()); + } + + /** + * Returns child expr if this expr is an implicit cast, otherwise returns 'this'. + */ + @Override + public Expr ignoreImplicitCast() { + if (isImplicit_) { + // we don't expect to see to consecutive implicit casts + Preconditions.checkState( + !(getChild(0) instanceof CastExpr) || !((CastExpr) getChild(0)).isImplicit()); + return getChild(0); + } else { + return this; + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj instanceof CastExpr) { + CastExpr other = (CastExpr) obj; + return isImplicit_ == other.isImplicit_ + && type_.equals(other.type_) + && super.equals(obj); + } + // Ignore implicit casts when comparing expr trees. + if (isImplicit_) return getChild(0).equals(obj); + return false; + } + + @Override + public Expr clone() { return new CastExpr(this); } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/CollectionStructType.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/CollectionStructType.java b/fe/src/main/java/org/apache/impala/analysis/CollectionStructType.java new file mode 100644 index 0000000..b45b856 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/CollectionStructType.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import java.util.ArrayList; + +import jline.internal.Preconditions; + +import com.cloudera.impala.catalog.ArrayType; +import com.cloudera.impala.catalog.MapType; +import com.cloudera.impala.catalog.ScalarType; +import com.cloudera.impala.catalog.StructField; +import com.cloudera.impala.catalog.StructType; +import com.cloudera.impala.catalog.Type; +import com.google.common.collect.Lists; + +/** + * Generated struct type describing the fields of a collection type + * that can be referenced in paths. + * + * Parent Type CollectionStructType + * array<i> --> struct<item:i,pos:bigint> + * map<k,v> --> struct<key:k,value:v> + */ +public class CollectionStructType extends StructType { + // True if this struct describes the fields of a map, + // false if it describes the fields of an array. + private final boolean isMapStruct_; + + // Field that can be skipped by implicit paths if its type is a struct. + private final StructField optionalField_; + + private CollectionStructType(ArrayList<StructField> fields, boolean isMapStruct) { + super(fields); + isMapStruct_ = isMapStruct; + if (isMapStruct_) { + optionalField_ = getField(Path.MAP_VALUE_FIELD_NAME); + } else { + optionalField_ = getField(Path.ARRAY_ITEM_FIELD_NAME); + } + Preconditions.checkNotNull(optionalField_); + } + + public static CollectionStructType createArrayStructType(ArrayType arrayType) { + Type itemType = arrayType.getItemType(); + ArrayList<StructField> fields = Lists.newArrayListWithCapacity(2); + // The item field name comes before the pos field name so that a path to the + // stored item corresponds to its physical path. + fields.add(new StructField(Path.ARRAY_ITEM_FIELD_NAME, itemType)); + fields.add(new StructField(Path.ARRAY_POS_FIELD_NAME, ScalarType.BIGINT)); + return new CollectionStructType(fields, false); + } + + public static CollectionStructType createMapStructType(MapType mapType) { + ArrayList<StructField> mapFields = Lists.newArrayListWithCapacity(2); + mapFields.add(new StructField(Path.MAP_KEY_FIELD_NAME, mapType.getKeyType())); + mapFields.add(new StructField(Path.MAP_VALUE_FIELD_NAME, mapType.getValueType())); + return new CollectionStructType(mapFields, true); + } + + public StructField getOptionalField() { return optionalField_; } + public boolean isMapStruct() { return isMapStruct_; } + public boolean isArrayStruct() { return !isMapStruct_; } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java b/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java new file mode 100644 index 0000000..8abed3e --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.authorization.Privilege; +import com.cloudera.impala.authorization.PrivilegeRequestBuilder; +import com.cloudera.impala.common.AnalysisException; +import com.google.common.base.Preconditions; + +/** + * Reference to a MAP or ARRAY collection type that implies its + * flattening during execution. + * TODO: We currently create a new slot in the root tuple descriptor for every + * relative collection ref, even if they have the same path. The BE currently relies on + * this behavior for setting collection slots to NULL after they have been unnested + * inside a SubplanNode. We could instead share the slot and the corresponding item tuple + * descriptor among all collection table refs with the same path. This change will + * require decoupling tuple descriptors from table aliases, i.e., a tuple descriptor + * should be able to back multiple aliases. + */ +public class CollectionTableRef extends TableRef { + ///////////////////////////////////////// + // BEGIN: Members that need to be reset() + + // Expr that returns the referenced collection. Typically a SlotRef into the + // parent scan's tuple. Result of analysis. Fully resolved against base tables. + private Expr collectionExpr_; + + // END: Members that need to be reset() + ///////////////////////////////////////// + + /** + * Create a CollectionTableRef from the original unresolved table ref as well as + * its resolved path. Sets table aliases and join-related attributes. + */ + public CollectionTableRef(TableRef tableRef, Path resolvedPath) { + super(tableRef); + Preconditions.checkState(resolvedPath.isResolved()); + resolvedPath_ = resolvedPath; + // Use the last path element as an implicit alias if no explicit alias was given. + if (hasExplicitAlias()) return; + String implicitAlias = rawPath_.get(rawPath_.size() - 1).toLowerCase(); + aliases_ = new String[] { implicitAlias }; + } + + /** + * C'tor for cloning. + */ + public CollectionTableRef(CollectionTableRef other) { + super(other); + collectionExpr_ = + (other.collectionExpr_ != null) ? other.collectionExpr_.clone() : null; + } + + /** + * Registers this collection table ref with the given analyzer and adds a slot + * descriptor for the materialized collection to be populated by parent scan. + * Also determines whether this collection table ref is correlated or not. + * + * If this function is called in the context of analyzing a WITH clause, then + * no slot is added to the parent descriptor so as to not pollute the analysis + * state of the parent block (the WITH-clause analyzer is discarded, and the + * parent analyzer could have an entirely different global state). + */ + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (isAnalyzed_) return; + desc_ = analyzer.registerTableRef(this); + if (isRelative() && !analyzer.isWithClause()) { + SlotDescriptor parentSlotDesc = analyzer.registerSlotRef(resolvedPath_); + parentSlotDesc.setItemTupleDesc(desc_); + collectionExpr_ = new SlotRef(parentSlotDesc); + // Must always be materialized to ensure the correct cardinality after unnesting. + analyzer.materializeSlots(collectionExpr_); + Analyzer parentAnalyzer = + analyzer.findAnalyzer(resolvedPath_.getRootDesc().getId()); + Preconditions.checkNotNull(parentAnalyzer); + if (parentAnalyzer != analyzer) { + TableRef parentRef = + parentAnalyzer.getTableRef(resolvedPath_.getRootDesc().getId()); + Preconditions.checkNotNull(parentRef); + // InlineViews are currently not supported as a parent ref. + Preconditions.checkState(!(parentRef instanceof InlineViewRef)); + correlatedTupleIds_.add(parentRef.getId()); + } + } + if (!isRelative()) { + // Register a table-level privilege request as well as a column-level privilege request + // for the collection-typed column. + Preconditions.checkNotNull(resolvedPath_.getRootTable()); + analyzer.registerAuthAndAuditEvent(resolvedPath_.getRootTable(), analyzer); + analyzer.registerPrivReq(new PrivilegeRequestBuilder(). + allOf(Privilege.SELECT).onColumn(desc_.getTableName().getDb(), + desc_.getTableName().getTbl(), desc_.getPath().getRawPath().get(0)) + .toRequest()); + } + isAnalyzed_ = true; + analyzeHints(analyzer); + + // TODO: For joins on nested collections some join ops can be simplified + // due to the containment relationship of the parent and child. For example, + // a FULL OUTER JOIN would become a LEFT OUTER JOIN, or a RIGHT SEMI JOIN + // would become an INNER or CROSS JOIN. + analyzeJoin(analyzer); + } + + @Override + public boolean isRelative() { + Preconditions.checkNotNull(resolvedPath_); + return resolvedPath_.getRootDesc() != null; + } + + public Expr getCollectionExpr() { return collectionExpr_; } + + @Override + protected CollectionTableRef clone() { return new CollectionTableRef(this); } + + @Override + public void reset() { + super.reset(); + collectionExpr_ = null; + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java b/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java new file mode 100644 index 0000000..e7a3170 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import java.util.List; + +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +import com.cloudera.impala.catalog.Type; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TColumn; +import com.cloudera.impala.util.MetaStoreUtil; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +/** + * Represents a column definition in a CREATE/ALTER TABLE/VIEW statement. + * Column definitions in CREATE/ALTER TABLE statements require a column type, + * whereas column definitions in CREATE/ALTER VIEW statements infer the column type from + * the corresponding view definition. All column definitions have an optional comment. + * Since a column definition refers a column stored in the Metastore, the column name + * must be valid according to the Metastore's rules (see @MetaStoreUtils). + */ +public class ColumnDef { + private final String colName_; + private String comment_; + + // Required in CREATE/ALTER TABLE stmts. Set to NULL in CREATE/ALTER VIEW stmts, + // for which we setType() after analyzing the defining view definition stmt. + private final TypeDef typeDef_; + private Type type_; + + public ColumnDef(String colName, TypeDef typeDef, String comment) { + colName_ = colName.toLowerCase(); + typeDef_ = typeDef; + comment_ = comment; + } + + /** + * Creates an analyzed ColumnDef from a Hive FieldSchema. Throws if the FieldSchema's + * type is not supported. + */ + private ColumnDef(FieldSchema fs) throws AnalysisException { + Type type = Type.parseColumnType(fs.getType()); + if (type == null) { + throw new AnalysisException(String.format( + "Unsupported type '%s' in Hive field schema '%s'", + fs.getType(), fs.getName())); + } + colName_ = fs.getName(); + typeDef_ = new TypeDef(type); + comment_ = fs.getComment(); + analyze(); + } + + public void setType(Type type) { type_ = type; } + public Type getType() { return type_; } + public TypeDef getTypeDef() { return typeDef_; } + public String getColName() { return colName_; } + public void setComment(String comment) { comment_ = comment; } + public String getComment() { return comment_; } + + public void analyze() throws AnalysisException { + // Check whether the column name meets the Metastore's requirements. + if (!MetaStoreUtils.validateName(colName_)) { + throw new AnalysisException("Invalid column/field name: " + colName_); + } + if (typeDef_ != null) { + typeDef_.analyze(null); + type_ = typeDef_.getType(); + } + Preconditions.checkNotNull(type_); + Preconditions.checkState(type_.isValid()); + // Check HMS constraints of type and comment. + String typeSql = type_.toSql(); + if (typeSql.length() > MetaStoreUtil.MAX_TYPE_NAME_LENGTH) { + throw new AnalysisException(String.format( + "Type of column '%s' exceeds maximum type length of %d characters:\n" + + "%s has %d characters.", colName_, MetaStoreUtil.MAX_TYPE_NAME_LENGTH, + typeSql, typeSql.length())); + } + if (comment_ != null && + comment_.length() > MetaStoreUtil.CREATE_MAX_COMMENT_LENGTH) { + throw new AnalysisException(String.format( + "Comment of column '%s' exceeds maximum length of %d characters:\n" + + "%s has %d characters.", colName_, MetaStoreUtil.CREATE_MAX_COMMENT_LENGTH, + comment_, comment_.length())); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(colName_); + if (type_ != null) { + sb.append(" " + type_.toString()); + } else { + sb.append(" " + typeDef_.toString()); + } + if (comment_ != null) sb.append(String.format(" COMMENT '%s'", comment_)); + return sb.toString(); + } + + public TColumn toThrift() { + TColumn col = new TColumn(new TColumn(getColName(), type_.toThrift())); + col.setComment(getComment()); + return col; + } + + public static List<ColumnDef> createFromFieldSchemas(List<FieldSchema> fieldSchemas) + throws AnalysisException { + List<ColumnDef> result = Lists.newArrayListWithCapacity(fieldSchemas.size()); + for (FieldSchema fs: fieldSchemas) result.add(new ColumnDef(fs)); + return result; + } + + public static List<FieldSchema> toFieldSchemas(List<ColumnDef> colDefs) { + return Lists.transform(colDefs, new Function<ColumnDef, FieldSchema>() { + public FieldSchema apply(ColumnDef colDef) { + Preconditions.checkNotNull(colDef.getType()); + return new FieldSchema(colDef.getColName(), colDef.getType().toSql(), + colDef.getComment()); + } + }); + } + +}
