http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/ArithmeticExpr.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/ArithmeticExpr.java b/fe/src/main/java/com/cloudera/impala/analysis/ArithmeticExpr.java deleted file mode 100644 index bf8b0ea..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/ArithmeticExpr.java +++ /dev/null @@ -1,268 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.catalog.Db; -import com.cloudera.impala.catalog.Function.CompareMode; -import com.cloudera.impala.catalog.ScalarFunction; -import com.cloudera.impala.catalog.ScalarType; -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TExprNodeType; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -public class ArithmeticExpr extends Expr { - enum OperatorPosition { - BINARY_INFIX, - UNARY_PREFIX, - UNARY_POSTFIX, - } - - enum Operator { - MULTIPLY("*", "multiply", OperatorPosition.BINARY_INFIX), - DIVIDE("/", "divide", OperatorPosition.BINARY_INFIX), - MOD("%", "mod", OperatorPosition.BINARY_INFIX), - INT_DIVIDE("DIV", "int_divide", OperatorPosition.BINARY_INFIX), - ADD("+", "add", OperatorPosition.BINARY_INFIX), - SUBTRACT("-", "subtract", OperatorPosition.BINARY_INFIX), - BITAND("&", "bitand", OperatorPosition.BINARY_INFIX), - BITOR("|", "bitor", OperatorPosition.BINARY_INFIX), - BITXOR("^", "bitxor", OperatorPosition.BINARY_INFIX), - BITNOT("~", "bitnot", OperatorPosition.UNARY_PREFIX), - FACTORIAL("!", "factorial", OperatorPosition.UNARY_POSTFIX); - - private final String description_; - private final String name_; - private final OperatorPosition pos_; - - private Operator(String description, String name, OperatorPosition pos) { - this.description_ = description; - this.name_ = name; - this.pos_ = pos; - } - - @Override - public String toString() { return description_; } - public String getName() { return name_; } - public OperatorPosition getPos() { return pos_; } - - public boolean isUnary() { - return pos_ == OperatorPosition.UNARY_PREFIX || - pos_ == OperatorPosition.UNARY_POSTFIX; - } - - public boolean isBinary() { - return pos_ == OperatorPosition.BINARY_INFIX; - } - } - - private final Operator op_; - - public Operator getOp() { return op_; } - - public ArithmeticExpr(Operator op, Expr e1, Expr e2) { - super(); - this.op_ = op; - Preconditions.checkNotNull(e1); - children_.add(e1); - Preconditions.checkArgument((op.isUnary() && e2 == null) || - (op.isBinary() && e2 != null)); - if (e2 != null) children_.add(e2); - } - - /** - * Copy c'tor used in clone(). - */ - protected ArithmeticExpr(ArithmeticExpr other) { - super(other); - op_ = other.op_; - } - - public static void initBuiltins(Db db) { - for (Type t: Type.getNumericTypes()) { - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.MULTIPLY.getName(), Lists.newArrayList(t, t), t)); - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.ADD.getName(), Lists.newArrayList(t, t), t)); - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.SUBTRACT.getName(), Lists.newArrayList(t, t), t)); - } - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.DIVIDE.getName(), - Lists.<Type>newArrayList(Type.DOUBLE, Type.DOUBLE), - Type.DOUBLE)); - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.DIVIDE.getName(), - Lists.<Type>newArrayList(Type.DECIMAL, Type.DECIMAL), - Type.DECIMAL)); - - /* - * MOD(), FACTORIAL(), BITAND(), BITOR(), BITXOR(), and BITNOT() are registered as - * builtins, see impala_functions.py - */ - for (Type t: Type.getIntegerTypes()) { - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.INT_DIVIDE.getName(), Lists.newArrayList(t, t), t)); - } - } - - @Override - public String debugString() { - return Objects.toStringHelper(this) - .add("op", op_) - .addValue(super.debugString()) - .toString(); - } - - @Override - public String toSqlImpl() { - if (children_.size() == 1) { - if (op_.getPos() == OperatorPosition.UNARY_PREFIX) { - return op_.toString() + getChild(0).toSql(); - } else { - assert(op_.getPos() == OperatorPosition.UNARY_POSTFIX); - return getChild(0).toSql() + op_.toString(); - } - } else { - Preconditions.checkState(children_.size() == 2); - return getChild(0).toSql() + " " + op_.toString() + " " + getChild(1).toSql(); - } - } - - @Override - protected void toThrift(TExprNode msg) { - msg.node_type = TExprNodeType.FUNCTION_CALL; - } - - /** - * Inserts a cast from child[childIdx] to targetType if one is necessary. - * Note this is different from Expr.castChild() since arithmetic for decimals - * the cast is handled as part of the operator and in general, the return type - * does not match the input types. - */ - void castChild(int childIdx, Type targetType) throws AnalysisException { - Type t = getChild(childIdx).getType(); - if (t.matchesType(targetType)) return; - if (targetType.isDecimal() && !t.isNull()) { - Preconditions.checkState(t.isScalarType()); - targetType = ((ScalarType) t).getMinResolutionDecimal(); - } - castChild(targetType, childIdx); - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - super.analyze(analyzer); - for (Expr child: children_) { - Expr operand = (Expr) child; - if (!operand.type_.isNumericType() && !operand.type_.isNull()) { - String errMsg = "Arithmetic operation requires numeric operands: " + toSql(); - if (operand instanceof Subquery && !operand.type_.isScalarType()) { - errMsg = "Subquery must return a single row: " + operand.toSql(); - } - throw new AnalysisException(errMsg); - } - } - - convertNumericLiteralsFromDecimal(analyzer); - Type t0 = getChild(0).getType(); - Type t1 = null; - if (op_.isUnary()) { - Preconditions.checkState(children_.size() == 1); - } else if (op_.isBinary()) { - Preconditions.checkState(children_.size() == 2); - t1 = getChild(1).getType(); - } - if (hasChildCosts()) evalCost_ = getChildCosts() + ARITHMETIC_OP_COST; - - String fnName = op_.getName(); - switch (op_) { - case ADD: - case SUBTRACT: - case DIVIDE: - case MULTIPLY: - case MOD: - type_ = TypesUtil.getArithmeticResultType(t0, t1, op_); - // If both of the children are null, we'll default to the DOUBLE version of the - // operator. This prevents the BE from seeing NULL_TYPE. - if (type_.isNull()) type_ = Type.DOUBLE; - break; - - case INT_DIVIDE: - case BITAND: - case BITOR: - case BITXOR: - if ((!t0.isNull() & !t0.isIntegerType()) || - (!t1.isNull() && !t1.isIntegerType())) { - throw new AnalysisException("Invalid non-integer argument to operation '" + - op_.toString() + "': " + this.toSql()); - } - type_ = Type.getAssignmentCompatibleType(t0, t1, false); - // If both of the children are null, we'll default to the INT version of the - // operator. This prevents the BE from seeing NULL_TYPE. - if (type_.isNull()) type_ = Type.INT; - Preconditions.checkState(type_.isIntegerType()); - break; - case BITNOT: - case FACTORIAL: - if (!t0.isNull() && !t0.isIntegerType()) { - throw new AnalysisException("'" + op_.toString() + "'" + - " operation only allowed on integer types: " + toSql()); - } - // Special-case NULL to resolve to the appropriate type. - if (op_ == Operator.BITNOT) { - if (t0.isNull()) castChild(0, Type.INT); - } else { - assert(op_ == Operator.FACTORIAL); - if (t0.isNull()) castChild(0, Type.BIGINT); - } - fn_ = getBuiltinFunction(analyzer, op_.getName(), collectChildReturnTypes(), - CompareMode.IS_SUPERTYPE_OF); - Preconditions.checkNotNull(fn_); - castForFunctionCall(false); - type_ = fn_.getReturnType(); - return; - default: - // the programmer forgot to deal with a case - Preconditions.checkState(false, - "Unknown arithmetic operation " + op_.toString() + " in: " + this.toSql()); - break; - } - - // Don't cast from decimal to decimal. The BE function can just handle this. - if (!(type_.isDecimal() && t0.isDecimal())) castChild(0, type_); - if (!(type_.isDecimal() && t1.isDecimal())) castChild(1, type_); - t0 = getChild(0).getType(); - t1 = getChild(1).getType(); - - fn_ = getBuiltinFunction(analyzer, fnName, collectChildReturnTypes(), - CompareMode.IS_IDENTICAL); - if (fn_ == null) { - Preconditions.checkState(false, String.format("No match " + - "for '%s' with operand types %s and %s", toSql(), t0, t1)); - } - Preconditions.checkState(type_.matchesType(fn_.getReturnType())); - } - - @Override - public Expr clone() { return new ArithmeticExpr(this); } -}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/AuthorizationStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AuthorizationStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/AuthorizationStmt.java deleted file mode 100644 index 4e88014..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/AuthorizationStmt.java +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.authorization.User; -import com.cloudera.impala.common.AnalysisException; -import com.google.common.base.Strings; - -/** - * Base class for all authorization statements - CREATE/DROP/SHOW ROLE, GRANT/REVOKE - * ROLE/privilege, etc. - */ -public class AuthorizationStmt extends StatementBase { - // Set during analysis - protected User requestingUser_; - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (!analyzer.getAuthzConfig().isEnabled()) { - throw new AnalysisException("Authorization is not enabled. To enable " + - "authorization restart Impala with the --server_name=<name> flag."); - } - if (analyzer.getAuthzConfig().isFileBasedPolicy()) { - throw new AnalysisException("Cannot execute authorization statement using a file" + - " based policy. To disable file based policies, restart Impala without the " + - "-authorization_policy_file flag set."); - } - if (Strings.isNullOrEmpty(analyzer.getUser().getName())) { - throw new AnalysisException("Cannot execute authorization statement with an " + - "empty username."); - } - requestingUser_ = analyzer.getUser(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/BaseTableRef.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/BaseTableRef.java b/fe/src/main/java/com/cloudera/impala/analysis/BaseTableRef.java deleted file mode 100644 index 69780e0..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/BaseTableRef.java +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.catalog.HdfsTable; -import com.cloudera.impala.catalog.Table; -import com.cloudera.impala.common.AnalysisException; -import com.google.common.base.Preconditions; - -/** - * Represents a reference to an actual table, such as an Hdfs or HBase table. - * BaseTableRefs are instantiated as a result of table resolution during analysis - * of a SelectStmt. - */ -public class BaseTableRef extends TableRef { - - /** - * Create a BaseTableRef from the original unresolved table ref as well as - * its resolved path. Sets table aliases and join-related attributes. - */ - public BaseTableRef(TableRef tableRef, Path resolvedPath) { - super(tableRef); - Preconditions.checkState(resolvedPath.isResolved()); - Preconditions.checkState(resolvedPath.isRootedAtTable()); - resolvedPath_ = resolvedPath; - // Set implicit aliases if no explicit one was given. - if (hasExplicitAlias()) return; - aliases_ = new String[] { - getTable().getTableName().toString().toLowerCase(), - getTable().getName().toLowerCase() }; - } - - /** - * C'tor for cloning. - */ - private BaseTableRef(BaseTableRef other) { - super(other); - } - - /** - * Register this table ref and then analyze the Join clause. - */ - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - analyzer.registerAuthAndAuditEvent(resolvedPath_.getRootTable(), analyzer); - desc_ = analyzer.registerTableRef(this); - isAnalyzed_ = true; - analyzeHints(analyzer); - analyzeJoin(analyzer); - analyzeSkipHeaderLineCount(); - } - - @Override - protected String tableRefToSql() { - // Enclose the alias in quotes if Hive cannot parse it without quotes. - // This is needed for view compatibility between Impala and Hive. - String aliasSql = null; - String alias = getExplicitAlias(); - if (alias != null) aliasSql = ToSqlUtils.getIdentSql(alias); - String tableHintsSql = ToSqlUtils.getPlanHintsSql(tableHints_); - return getTable().getTableName().toSql() + - ((aliasSql != null) ? " " + aliasSql : "") + - (tableHintsSql != "" ? " " + tableHintsSql : ""); - } - - public String debugString() { return tableRefToSql(); } - @Override - protected TableRef clone() { return new BaseTableRef(this); } - - /** - * Analyze the 'skip.header.line.count' property. - */ - private void analyzeSkipHeaderLineCount() throws AnalysisException { - Table table = getTable(); - if (!(table instanceof HdfsTable)) return; - HdfsTable hdfsTable = (HdfsTable)table; - - StringBuilder error = new StringBuilder(); - hdfsTable.parseSkipHeaderLineCount(error); - if (error.length() > 0) throw new AnalysisException(error.toString()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/BetweenPredicate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/BetweenPredicate.java b/fe/src/main/java/com/cloudera/impala/analysis/BetweenPredicate.java deleted file mode 100644 index d76a4c6..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/BetweenPredicate.java +++ /dev/null @@ -1,158 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.ArrayList; -import java.util.List; - -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TExprNode; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -/** - * Class describing between predicates. After successful analysis, we rewrite - * the between predicate to a conjunctive/disjunctive compound predicate - * to be handed to the backend. - */ -public class BetweenPredicate extends Predicate { - - private final boolean isNotBetween_; - - // After successful analysis, we rewrite this between predicate - // into a conjunctive/disjunctive compound predicate. - private CompoundPredicate rewrittenPredicate_; - - // Children of the BetweenPredicate, since this.children should hold the children - // of the rewritten predicate to make sure toThrift() picks up the right ones. - private ArrayList<Expr> originalChildren_ = Lists.newArrayList(); - - // First child is the comparison expr which should be in [lowerBound, upperBound]. - public BetweenPredicate(Expr compareExpr, Expr lowerBound, Expr upperBound, - boolean isNotBetween) { - originalChildren_.add(compareExpr); - originalChildren_.add(lowerBound); - originalChildren_.add(upperBound); - this.isNotBetween_ = isNotBetween; - } - - /** - * Copy c'tor used in clone(). - */ - protected BetweenPredicate(BetweenPredicate other) { - super(other); - isNotBetween_ = other.isNotBetween_; - originalChildren_ = Expr.cloneList(other.originalChildren_); - if (other.rewrittenPredicate_ != null) { - rewrittenPredicate_ = (CompoundPredicate) other.rewrittenPredicate_.clone(); - } - } - - public CompoundPredicate getRewrittenPredicate() { - Preconditions.checkState(isAnalyzed_); - return rewrittenPredicate_; - } - public ArrayList<Expr> getOriginalChildren() { return originalChildren_; } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - super.analyze(analyzer); - if (originalChildren_.get(0) instanceof Subquery && - (originalChildren_.get(1) instanceof Subquery || - originalChildren_.get(2) instanceof Subquery)) { - throw new AnalysisException("Comparison between subqueries is not " + - "supported in a between predicate: " + toSqlImpl()); - } - analyzer.castAllToCompatibleType(originalChildren_); - - // Rewrite between predicate into a conjunctive/disjunctive compound predicate. - if (isNotBetween_) { - // Rewrite into disjunction. - Predicate lower = new BinaryPredicate(BinaryPredicate.Operator.LT, - originalChildren_.get(0), originalChildren_.get(1)); - Predicate upper = new BinaryPredicate(BinaryPredicate.Operator.GT, - originalChildren_.get(0), originalChildren_.get(2)); - rewrittenPredicate_ = - new CompoundPredicate(CompoundPredicate.Operator.OR, lower, upper); - } else { - // Rewrite into conjunction. - Predicate lower = new BinaryPredicate(BinaryPredicate.Operator.GE, - originalChildren_.get(0), originalChildren_.get(1)); - Predicate upper = new BinaryPredicate(BinaryPredicate.Operator.LE, - originalChildren_.get(0), originalChildren_.get(2)); - rewrittenPredicate_ = - new CompoundPredicate(CompoundPredicate.Operator.AND, lower, upper); - } - - try { - rewrittenPredicate_.analyze(analyzer); - fn_ = rewrittenPredicate_.fn_; - } catch (AnalysisException e) { - // We should have already guaranteed that analysis will succeed. - Preconditions.checkState(false, "Analysis failed in rewritten between predicate"); - } - - // Make sure toThrift() picks up the children of the rewritten predicate. - children_ = rewrittenPredicate_.getChildren(); - // Since the only child is a CompoundPredicate expressing the comparison, - // the cost of the comparison is fully captured by the children's cost. - evalCost_ = getChildCosts(); - isAnalyzed_ = true; - } - - @Override - public List<Expr> getConjuncts() { - return rewrittenPredicate_.getConjuncts(); - } - - @Override - protected void toThrift(TExprNode msg) { - rewrittenPredicate_.toThrift(msg); - } - - @Override - public String toSqlImpl() { - String notStr = (isNotBetween_) ? "NOT " : ""; - return originalChildren_.get(0).toSql() + " " + notStr + "BETWEEN " + - originalChildren_.get(1).toSql() + " AND " + originalChildren_.get(2).toSql(); - } - - /** - * Also substitute the exprs in originalChildren when cloning. - */ - @Override - protected Expr substituteImpl(ExprSubstitutionMap smap, Analyzer analyzer) - throws AnalysisException { - BetweenPredicate clone = (BetweenPredicate) super.substituteImpl(smap, analyzer); - Preconditions.checkNotNull(clone); - clone.originalChildren_ = - Expr.substituteList(originalChildren_, smap, analyzer, false); - return clone; - } - - @Override - public Expr clone() { return new BetweenPredicate(this); } - - @Override - public Expr reset() { - super.reset(); - originalChildren_ = Expr.resetList(originalChildren_); - return this; - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/BinaryPredicate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/BinaryPredicate.java b/fe/src/main/java/com/cloudera/impala/analysis/BinaryPredicate.java deleted file mode 100644 index 35d03e1..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/BinaryPredicate.java +++ /dev/null @@ -1,388 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.cloudera.impala.catalog.Db; -import com.cloudera.impala.catalog.Function.CompareMode; -import com.cloudera.impala.catalog.ScalarFunction; -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.Pair; -import com.cloudera.impala.common.Reference; -import com.cloudera.impala.extdatasource.thrift.TComparisonOp; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TExprNodeType; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicates; -import com.google.common.collect.Lists; - -/** - * Most predicates with two operands. - * - */ -public class BinaryPredicate extends Predicate { - private final static Logger LOG = LoggerFactory.getLogger(BinaryPredicate.class); - - // true if this BinaryPredicate is inferred from slot equivalences, false otherwise. - private boolean isInferred_ = false; - - public enum Operator { - EQ("=", "eq", TComparisonOp.EQ), - NE("!=", "ne", TComparisonOp.NE), - LE("<=", "le", TComparisonOp.LE), - GE(">=", "ge", TComparisonOp.GE), - LT("<", "lt", TComparisonOp.LT), - GT(">", "gt", TComparisonOp.GT), - DISTINCT_FROM("IS DISTINCT FROM", "distinctfrom", TComparisonOp.DISTINCT_FROM), - NOT_DISTINCT("IS NOT DISTINCT FROM", "notdistinct", TComparisonOp.NOT_DISTINCT), - // Same as EQ, except it returns True if the rhs is NULL. There is no backend - // function for this. The functionality is embedded in the hash-join - // implementation. - NULL_MATCHING_EQ("=", "null_matching_eq", TComparisonOp.EQ); - - private final String description_; - private final String name_; - private final TComparisonOp thriftOp_; - - private Operator(String description, String name, TComparisonOp thriftOp) { - this.description_ = description; - this.name_ = name; - this.thriftOp_ = thriftOp; - } - - @Override - public String toString() { return description_; } - public String getName() { return name_; } - public TComparisonOp getThriftOp() { return thriftOp_; } - public boolean isEquivalence() { return this == EQ || this == NOT_DISTINCT; } - - public Operator converse() { - switch (this) { - case EQ: return EQ; - case NE: return NE; - case LE: return GE; - case GE: return LE; - case LT: return GT; - case GT: return LT; - case DISTINCT_FROM: return DISTINCT_FROM; - case NOT_DISTINCT: return NOT_DISTINCT; - case NULL_MATCHING_EQ: - throw new IllegalStateException("Not implemented"); - default: throw new IllegalStateException("Invalid operator"); - } - } - } - - public static void initBuiltins(Db db) { - for (Type t: Type.getSupportedTypes()) { - if (t.isNull()) continue; // NULL is handled through type promotion. - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.EQ.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.NE.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.LE.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.GE.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.LT.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - db.addBuiltin(ScalarFunction.createBuiltinOperator( - Operator.GT.getName(), Lists.newArrayList(t, t), Type.BOOLEAN)); - } - } - - /** - * Normalizes a 'predicate' consisting of an uncast SlotRef and a constant Expr into - * the following form: <SlotRef> <Op> <LiteralExpr> - * If 'predicate' cannot be expressed in this way, null is returned. - */ - public static BinaryPredicate normalizeSlotRefComparison(BinaryPredicate predicate, - Analyzer analyzer) { - SlotRef ref = null; - if (predicate.getChild(0) instanceof SlotRef) { - ref = (SlotRef) predicate.getChild(0); - } else if (predicate.getChild(1) instanceof SlotRef) { - ref = (SlotRef) predicate.getChild(1); - } - - if (ref == null) return null; - if (ref != predicate.getChild(0)) { - Preconditions.checkState(ref == predicate.getChild(1)); - predicate = new BinaryPredicate(predicate.getOp().converse(), ref, - predicate.getChild(0)); - predicate.analyzeNoThrow(analyzer); - } - - try { - predicate.foldConstantChildren(analyzer); - } catch (AnalysisException ex) { - // Throws if the expression cannot be evaluated by the BE. - return null; - } - predicate.analyzeNoThrow(analyzer); - if (!(predicate.getChild(1) instanceof LiteralExpr)) return null; - return predicate; - } - - private Operator op_; - - public Operator getOp() { return op_; } - public void setOp(Operator op) { op_ = op; } - - public BinaryPredicate(Operator op, Expr e1, Expr e2) { - super(); - this.op_ = op; - Preconditions.checkNotNull(e1); - children_.add(e1); - Preconditions.checkNotNull(e2); - children_.add(e2); - } - - protected BinaryPredicate(BinaryPredicate other) { - super(other); - op_ = other.op_; - isInferred_ = other.isInferred_; - } - - public boolean isNullMatchingEq() { return op_ == Operator.NULL_MATCHING_EQ; } - - public boolean isInferred() { return isInferred_; } - public void setIsInferred() { isInferred_ = true; } - - @Override - public String toSqlImpl() { - return getChild(0).toSql() + " " + op_.toString() + " " + getChild(1).toSql(); - } - - @Override - protected void toThrift(TExprNode msg) { - Preconditions.checkState(children_.size() == 2); - // Cannot serialize a nested predicate. - Preconditions.checkState(!contains(Subquery.class)); - // This check is important because we often clone and/or evaluate predicates, - // and it's easy to get the casting logic wrong, e.g., cloned predicates - // with expr substitutions need to be re-analyzed with reanalyze(). - Preconditions.checkState(getChild(0).getType().getPrimitiveType() == - getChild(1).getType().getPrimitiveType(), - "child 0 type: " + getChild(0).getType() + - " child 1 type: " + getChild(1).getType()); - msg.node_type = TExprNodeType.FUNCTION_CALL; - } - - @Override - public String debugString() { - return Objects.toStringHelper(this) - .add("op", op_) - .addValue(super.debugString()) - .toString(); - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - super.analyze(analyzer); - - convertNumericLiteralsFromDecimal(analyzer); - String opName = op_.getName().equals("null_matching_eq") ? "eq" : op_.getName(); - fn_ = getBuiltinFunction(analyzer, opName, collectChildReturnTypes(), - CompareMode.IS_NONSTRICT_SUPERTYPE_OF); - if (fn_ == null) { - // Construct an appropriate error message and throw an AnalysisException. - String errMsg = "operands of type " + getChild(0).getType().toSql() + " and " + - getChild(1).getType().toSql() + " are not comparable: " + toSql(); - - // Check if any of the children is a Subquery that does not return a - // scalar. - for (Expr expr: children_) { - if (expr instanceof Subquery && !expr.getType().isScalarType()) { - errMsg = "Subquery must return a single row: " + expr.toSql(); - break; - } - } - - throw new AnalysisException(errMsg); - } - Preconditions.checkState(fn_.getReturnType().isBoolean()); - - ArrayList<Expr> subqueries = Lists.newArrayList(); - collectAll(Predicates.instanceOf(Subquery.class), subqueries); - if (subqueries.size() > 1) { - // TODO Remove that restriction when we add support for independent subquery - // evaluation. - throw new AnalysisException("Multiple subqueries are not supported in binary " + - "predicates: " + toSql()); - } - if (contains(ExistsPredicate.class)) { - throw new AnalysisException("EXISTS subquery predicates are not " + - "supported in binary predicates: " + toSql()); - } - - List<InPredicate> inPredicates = Lists.newArrayList(); - collect(InPredicate.class, inPredicates); - for (InPredicate inPredicate: inPredicates) { - if (inPredicate.contains(Subquery.class)) { - throw new AnalysisException("IN subquery predicates are not supported in " + - "binary predicates: " + toSql()); - } - } - - // Don't perform any casting for predicates with subqueries here. Any casting - // required will be performed when the subquery is unnested. - if (!contains(Subquery.class)) castForFunctionCall(true); - - // Determine selectivity - // TODO: Compute selectivity for nested predicates. - // TODO: Improve estimation using histograms. - Reference<SlotRef> slotRefRef = new Reference<SlotRef>(); - if ((op_ == Operator.EQ || op_ == Operator.NOT_DISTINCT) - && isSingleColumnPredicate(slotRefRef, null)) { - long distinctValues = slotRefRef.getRef().getNumDistinctValues(); - if (distinctValues > 0) { - selectivity_ = 1.0 / distinctValues; - selectivity_ = Math.max(0, Math.min(1, selectivity_)); - } - } - - // Compute cost. - if (hasChildCosts()) { - if (getChild(0).getType().isFixedLengthType()) { - evalCost_ = getChildCosts() + BINARY_PREDICATE_COST; - } else if (getChild(0).getType().isStringType()) { - evalCost_ = getChildCosts() + - (float) (getAvgStringLength(getChild(0)) + getAvgStringLength(getChild(1)) * - BINARY_PREDICATE_COST); - } else { - //TODO(tmarshall): Handle other var length types here. - evalCost_ = getChildCosts() + VAR_LEN_BINARY_PREDICATE_COST; - } - } - } - - /** - * If predicate is of the form "<slotref> <op> <expr>", returns expr, - * otherwise returns null. Slotref may be wrapped in a CastExpr. - * TODO: revisit CAST handling at the caller - */ - public Expr getSlotBinding(SlotId id) { - // check left operand - SlotRef slotRef = getChild(0).unwrapSlotRef(false); - if (slotRef != null && slotRef.getSlotId() == id) return getChild(1); - // check right operand - slotRef = getChild(1).unwrapSlotRef(false); - if (slotRef != null && slotRef.getSlotId() == id) return getChild(0); - return null; - } - - /** - * If e is an equality predicate between two slots that only require implicit - * casts, returns those two slots; otherwise returns null. - */ - public static Pair<SlotId, SlotId> getEqSlots(Expr e) { - if (!(e instanceof BinaryPredicate)) return null; - return ((BinaryPredicate) e).getEqSlots(); - } - - /** - * If this is an equality predicate between two slots that only require implicit - * casts, returns those two slots; otherwise returns null. - */ - @Override - public Pair<SlotId, SlotId> getEqSlots() { - if (op_ != Operator.EQ) return null; - SlotRef lhs = getChild(0).unwrapSlotRef(true); - if (lhs == null) return null; - SlotRef rhs = getChild(1).unwrapSlotRef(true); - if (rhs == null) return null; - return new Pair<SlotId, SlotId>(lhs.getSlotId(), rhs.getSlotId()); - } - - /** - * If predicate is of the form "<SlotRef> op <Expr>" or "<Expr> op <SlotRef>", - * returns the SlotRef, otherwise returns null. - */ - @Override - public SlotRef getBoundSlot() { - SlotRef slotRef = getChild(0).unwrapSlotRef(true); - if (slotRef != null) return slotRef; - return getChild(1).unwrapSlotRef(true); - } - - /** - * Negates a BinaryPredicate. - */ - @Override - public Expr negate() { - Operator newOp = null; - switch (op_) { - case EQ: - newOp = Operator.NE; - break; - case NE: - newOp = Operator.EQ; - break; - case LT: - newOp = Operator.GE; - break; - case LE: - newOp = Operator.GT; - break; - case GE: - newOp = Operator.LT; - break; - case GT: - newOp = Operator.LE; - break; - case DISTINCT_FROM: - newOp = Operator.NOT_DISTINCT; - break; - case NOT_DISTINCT: - newOp = Operator.DISTINCT_FROM; - break; - case NULL_MATCHING_EQ: - throw new IllegalStateException("Not implemented"); - } - return new BinaryPredicate(newOp, getChild(0), getChild(1)); - } - - /** - * Swaps the first with the second child in-place. Only valid to call for - * equivalence and not equal predicates. - */ - public void reverse() { - Preconditions.checkState(op_.isEquivalence() || op_ == Operator.NE); - Collections.swap(children_, 0, 1); - } - - @Override - public boolean equals(Object obj) { - if (!super.equals(obj)) return false; - BinaryPredicate other = (BinaryPredicate) obj; - return op_.equals(other.op_); - } - - @Override - public Expr clone() { return new BinaryPredicate(this); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/BoolLiteral.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/BoolLiteral.java b/fe/src/main/java/com/cloudera/impala/analysis/BoolLiteral.java deleted file mode 100644 index 03b2b1f..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/BoolLiteral.java +++ /dev/null @@ -1,113 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TBoolLiteral; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TExprNodeType; -import com.google.common.base.Objects; - -public class BoolLiteral extends LiteralExpr { - private final boolean value_; - - public BoolLiteral(boolean value) { - this.value_ = value; - type_ = Type.BOOLEAN; - evalCost_ = LITERAL_COST; - } - - public BoolLiteral(String value) throws AnalysisException { - type_ = Type.BOOLEAN; - evalCost_ = LITERAL_COST; - if (value.toLowerCase().equals("true")) { - this.value_ = true; - } else if (value.toLowerCase().equals("false")) { - this.value_ = false; - } else { - throw new AnalysisException("invalid BOOLEAN literal: " + value); - } - } - - /** - * Copy c'tor used in clone. - */ - protected BoolLiteral(BoolLiteral other) { - super(other); - value_ = other.value_; - } - - @Override - public String debugString() { - return Objects.toStringHelper(this) - .add("value", value_) - .toString(); - } - - @Override - public boolean equals(Object obj) { - if (!super.equals(obj)) { - return false; - } - return ((BoolLiteral) obj).value_ == value_; - } - - @Override - public int hashCode() { return value_ ? 1 : 0; } - - public boolean getValue() { return value_; } - - @Override - public String toSqlImpl() { - return getStringValue(); - } - - @Override - public String getStringValue() { - return value_ ? "TRUE" : "FALSE"; - } - - @Override - protected void toThrift(TExprNode msg) { - msg.node_type = TExprNodeType.BOOL_LITERAL; - msg.bool_literal = new TBoolLiteral(value_); - } - - @Override - protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { - if (targetType.equals(this.type_)) { - return this; - } else { - return new CastExpr(targetType, this); - } - } - - @Override - public int compareTo(LiteralExpr o) { - int ret = super.compareTo(o); - if (ret != 0) return ret; - BoolLiteral other = (BoolLiteral) o; - if (value_ && !other.getValue()) return 1; - if (!value_ && other.getValue()) return -1; - return 0; - } - - @Override - public Expr clone() { return new BoolLiteral(this); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/CaseExpr.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/CaseExpr.java b/fe/src/main/java/com/cloudera/impala/analysis/CaseExpr.java deleted file mode 100644 index bd3ec83..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/CaseExpr.java +++ /dev/null @@ -1,379 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.List; - -import com.cloudera.impala.catalog.Db; -import com.cloudera.impala.catalog.Function.CompareMode; -import com.cloudera.impala.catalog.PrimitiveType; -import com.cloudera.impala.catalog.ScalarFunction; -import com.cloudera.impala.catalog.ScalarType; -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TCaseExpr; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TExprNodeType; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -/** - * CASE and DECODE are represented using this class. The backend implementation is - * always the "case" function. - * - * The internal representation of - * CASE [expr] WHEN expr THEN expr [WHEN expr THEN expr ...] [ELSE expr] END - * Each When/Then is stored as two consecutive children (whenExpr, thenExpr). If a case - * expr is given then it is the first child. If an else expr is given then it is the - * last child. - * - * The internal representation of - * DECODE(expr, key_expr, val_expr [, key_expr, val_expr ...] [, default_val_expr]) - * has a pair of children for each pair of key/val_expr and an additional child if the - * default_val_expr was given. The first child represents the comparison of expr to - * key_expr. Decode has three forms: - * 1) DECODE(expr, null_literal, val_expr) - - * child[0] = IsNull(expr) - * 2) DECODE(expr, non_null_literal, val_expr) - - * child[0] = Eq(expr, literal) - * 3) DECODE(expr1, expr2, val_expr) - - * child[0] = Or(And(IsNull(expr1), IsNull(expr2)), Eq(expr1, expr2)) - * The children representing val_expr (child[1]) and default_val_expr (child[2]) are - * simply the exprs themselves. - * - * Example of equivalent CASE for DECODE(foo, 'bar', 1, col, 2, NULL, 3, 4): - * CASE - * WHEN foo = 'bar' THEN 1 -- no need for IS NULL check - * WHEN foo IS NULL AND col IS NULL OR foo = col THEN 2 - * WHEN foo IS NULL THEN 3 -- no need for equality check - * ELSE 4 - * END - */ -public class CaseExpr extends Expr { - - // Set if constructed from a DECODE, null otherwise. - private FunctionCallExpr decodeExpr_; - - private boolean hasCaseExpr_; - private boolean hasElseExpr_; - - public CaseExpr(Expr caseExpr, List<CaseWhenClause> whenClauses, Expr elseExpr) { - super(); - if (caseExpr != null) { - children_.add(caseExpr); - hasCaseExpr_ = true; - } - for (CaseWhenClause whenClause: whenClauses) { - Preconditions.checkNotNull(whenClause.getWhenExpr()); - children_.add(whenClause.getWhenExpr()); - Preconditions.checkNotNull(whenClause.getThenExpr()); - children_.add(whenClause.getThenExpr()); - } - if (elseExpr != null) { - children_.add(elseExpr); - hasElseExpr_ = true; - } - } - - /** - * Constructs an equivalent CaseExpr representation. - * - * The DECODE behavior is basically the same as the hasCaseExpr_ version of CASE. - * Though there is one difference. NULLs are considered equal when comparing the - * argument to be decoded with the candidates. This differences is for compatibility - * with Oracle. http://docs.oracle.com/cd/B19306_01/server.102/b14200/functions040.htm. - * To account for the difference, the CASE representation will use the non-hasCaseExpr_ - * version. - * - * The return type of DECODE differs from that of Oracle when the third argument is - * the NULL literal. In Oracle the return type is STRING. In Impala the return type is - * determined by the implicit casting rules (i.e. it's not necessarily a STRING). This - * is done so seemingly normal usages such as DECODE(int_col, tinyint_col, NULL, - * bigint_col) will avoid type check errors (STRING incompatible with BIGINT). - */ - public CaseExpr(FunctionCallExpr decodeExpr) { - super(); - decodeExpr_ = decodeExpr; - hasCaseExpr_ = false; - - int childIdx = 0; - Expr encoded = null; - Expr encodedIsNull = null; - if (!decodeExpr.getChildren().isEmpty()) { - encoded = decodeExpr.getChild(childIdx++); - encodedIsNull = new IsNullPredicate(encoded, false); - } - - // Add the key_expr/val_expr pairs - while (childIdx + 2 <= decodeExpr.getChildren().size()) { - Expr candidate = decodeExpr.getChild(childIdx++); - if (candidate.isLiteral()) { - if (candidate.isNullLiteral()) { - // An example case is DECODE(foo, NULL, bar), since NULLs are considered - // equal, this becomes CASE WHEN foo IS NULL THEN bar END. - children_.add(encodedIsNull); - } else { - children_.add(new BinaryPredicate( - BinaryPredicate.Operator.EQ, encoded, candidate)); - } - } else { - children_.add(new CompoundPredicate(CompoundPredicate.Operator.OR, - new CompoundPredicate(CompoundPredicate.Operator.AND, - encodedIsNull, new IsNullPredicate(candidate, false)), - new BinaryPredicate(BinaryPredicate.Operator.EQ, encoded, candidate))); - } - - // Add the value - children_.add(decodeExpr.getChild(childIdx++)); - } - - // Add the default value - if (childIdx < decodeExpr.getChildren().size()) { - hasElseExpr_ = true; - children_.add(decodeExpr.getChild(childIdx)); - } - } - - /** - * Copy c'tor used in clone(). - */ - protected CaseExpr(CaseExpr other) { - super(other); - decodeExpr_ = other.decodeExpr_; - hasCaseExpr_ = other.hasCaseExpr_; - hasElseExpr_ = other.hasElseExpr_; - } - - public static void initBuiltins(Db db) { - for (Type t: Type.getSupportedTypes()) { - if (t.isNull()) continue; - if (t.isScalarType(PrimitiveType.CHAR)) continue; - // TODO: case is special and the signature cannot be represented. - // It is alternating varargs - // e.g. case(bool, type, bool type, bool type, etc). - // Instead we just add a version for each of the return types - // e.g. case(BOOLEAN), case(INT), etc - db.addBuiltin(ScalarFunction.createBuiltinOperator( - "case", "", Lists.newArrayList(t), t)); - // Same for DECODE - db.addBuiltin(ScalarFunction.createBuiltinOperator( - "decode", "", Lists.newArrayList(t), t)); - } - } - - @Override - public boolean equals(Object obj) { - if (!super.equals(obj)) return false; - CaseExpr expr = (CaseExpr) obj; - return hasCaseExpr_ == expr.hasCaseExpr_ - && hasElseExpr_ == expr.hasElseExpr_ - && isDecode() == expr.isDecode(); - } - - @Override - public String toSqlImpl() { - return (decodeExpr_ == null) ? toCaseSql() : decodeExpr_.toSqlImpl(); - } - - @VisibleForTesting - String toCaseSql() { - StringBuilder output = new StringBuilder("CASE"); - int childIdx = 0; - if (hasCaseExpr_) { - output.append(" " + children_.get(childIdx++).toSql()); - } - while (childIdx + 2 <= children_.size()) { - output.append(" WHEN " + children_.get(childIdx++).toSql()); - output.append(" THEN " + children_.get(childIdx++).toSql()); - } - if (hasElseExpr_) { - output.append(" ELSE " + children_.get(children_.size() - 1).toSql()); - } - output.append(" END"); - return output.toString(); - } - - @Override - protected void toThrift(TExprNode msg) { - msg.node_type = TExprNodeType.CASE_EXPR; - msg.case_expr = new TCaseExpr(hasCaseExpr_, hasElseExpr_); - } - - private void castCharToString(int childIndex) throws AnalysisException { - if (children_.get(childIndex).getType().isScalarType(PrimitiveType.CHAR)) { - children_.set(childIndex, children_.get(childIndex).castTo(ScalarType.STRING)); - } - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - super.analyze(analyzer); - - if (isDecode()) { - Preconditions.checkState(!hasCaseExpr_); - // decodeExpr_.analyze() would fail validating function existence. The complex - // vararg signature is currently unsupported. - FunctionCallExpr.validateScalarFnParams(decodeExpr_.getParams()); - if (decodeExpr_.getChildren().size() < 3) { - throw new AnalysisException("DECODE in '" + toSql() + "' requires at least 3 " - + "arguments."); - } - } - - // Since we have no BE implementation of a CaseExpr with CHAR types, - // we cast the CHAR-typed whenExprs and caseExprs to STRING, - // TODO: This casting is not always correct and needs to be fixed, see IMPALA-1652. - - // Keep track of maximum compatible type of case expr and all when exprs. - Type whenType = null; - // Keep track of maximum compatible type of else expr and all then exprs. - Type returnType = null; - // Remember last of these exprs for error reporting. - Expr lastCompatibleThenExpr = null; - Expr lastCompatibleWhenExpr = null; - int loopEnd = children_.size(); - if (hasElseExpr_) { - --loopEnd; - } - int loopStart; - Expr caseExpr = null; - // Set loop start, and initialize returnType as type of castExpr. - if (hasCaseExpr_) { - loopStart = 1; - castCharToString(0); - caseExpr = children_.get(0); - caseExpr.analyze(analyzer); - whenType = caseExpr.getType(); - lastCompatibleWhenExpr = children_.get(0); - } else { - whenType = Type.BOOLEAN; - loopStart = 0; - } - - // Go through when/then exprs and determine compatible types. - for (int i = loopStart; i < loopEnd; i += 2) { - castCharToString(i); - Expr whenExpr = children_.get(i); - if (hasCaseExpr_) { - // Determine maximum compatible type of the case expr, - // and all when exprs seen so far. We will add casts to them at the very end. - whenType = analyzer.getCompatibleType(whenType, - lastCompatibleWhenExpr, whenExpr); - lastCompatibleWhenExpr = whenExpr; - } else { - // If no case expr was given, then the when exprs should always return - // boolean or be castable to boolean. - if (!Type.isImplicitlyCastable(whenExpr.getType(), Type.BOOLEAN, false)) { - Preconditions.checkState(isCase()); - throw new AnalysisException("When expr '" + whenExpr.toSql() + "'" + - " is not of type boolean and not castable to type boolean."); - } - // Add a cast if necessary. - if (!whenExpr.getType().isBoolean()) castChild(Type.BOOLEAN, i); - } - // Determine maximum compatible type of the then exprs seen so far. - // We will add casts to them at the very end. - Expr thenExpr = children_.get(i + 1); - returnType = analyzer.getCompatibleType(returnType, - lastCompatibleThenExpr, thenExpr); - lastCompatibleThenExpr = thenExpr; - } - if (hasElseExpr_) { - Expr elseExpr = children_.get(children_.size() - 1); - returnType = analyzer.getCompatibleType(returnType, - lastCompatibleThenExpr, elseExpr); - } - - // Make sure BE doesn't see TYPE_NULL by picking an arbitrary type - if (whenType.isNull()) whenType = ScalarType.BOOLEAN; - if (returnType.isNull()) returnType = ScalarType.BOOLEAN; - - // Add casts to case expr to compatible type. - if (hasCaseExpr_) { - // Cast case expr. - if (!children_.get(0).type_.equals(whenType)) { - castChild(whenType, 0); - } - // Add casts to when exprs to compatible type. - for (int i = loopStart; i < loopEnd; i += 2) { - if (!children_.get(i).type_.equals(whenType)) { - castChild(whenType, i); - } - } - } - // Cast then exprs to compatible type. - for (int i = loopStart + 1; i < children_.size(); i += 2) { - if (!children_.get(i).type_.equals(returnType)) { - castChild(returnType, i); - } - } - // Cast else expr to compatible type. - if (hasElseExpr_) { - if (!children_.get(children_.size() - 1).type_.equals(returnType)) { - castChild(returnType, children_.size() - 1); - } - } - - // Do the function lookup just based on the whenType. - Type[] args = new Type[1]; - args[0] = whenType; - fn_ = getBuiltinFunction(analyzer, "case", args, - CompareMode.IS_NONSTRICT_SUPERTYPE_OF); - Preconditions.checkNotNull(fn_); - type_ = returnType; - - // Compute cost as the sum of evaluating all of the WHEN exprs, plus - // the max of the THEN/ELSE exprs. - float maxThenCost = 0; - float whenCosts = 0; - boolean hasChildCosts = true; - for (int i = 0; i < children_.size(); ++i) { - if (!getChild(i).hasCost()) { - hasChildCosts = false; - break; - } - - if (hasCaseExpr_ && i % 2 == 1) { - // This child is a WHEN expr. BINARY_PREDICATE_COST accounts for the cost of - // comparing the CASE expr to the WHEN expr. - whenCosts += getChild(0).getCost() + getChild(i).getCost() + - BINARY_PREDICATE_COST; - } else if (!hasCaseExpr_ && i % 2 == 0) { - // This child is a WHEN expr. - whenCosts += getChild(i).getCost(); - } else if (i != 0) { - // This child is a THEN or ELSE expr. - float thenCost = getChild(i).getCost(); - if (thenCost > maxThenCost) maxThenCost = thenCost; - } - } - if (hasChildCosts) { - evalCost_ = whenCosts + maxThenCost; - } - } - - private boolean isCase() { return !isDecode(); } - private boolean isDecode() { return decodeExpr_ != null; } - public boolean hasCaseExpr() { return hasCaseExpr_; } - public boolean hasElseExpr() { return hasElseExpr_; } - - @Override - public Expr clone() { return new CaseExpr(this); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/CaseWhenClause.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/CaseWhenClause.java b/fe/src/main/java/com/cloudera/impala/analysis/CaseWhenClause.java deleted file mode 100644 index 8b1433e..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/CaseWhenClause.java +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - - -/** - * captures info of a single WHEN expr THEN expr clause. - * - */ -class CaseWhenClause { - private final Expr whenExpr_; - private final Expr thenExpr_; - - public CaseWhenClause(Expr whenExpr, Expr thenExpr) { - super(); - this.whenExpr_ = whenExpr; - this.thenExpr_ = thenExpr; - } - - public Expr getWhenExpr() { - return whenExpr_; - } - - public Expr getThenExpr() { - return thenExpr_; - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/CastExpr.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/CastExpr.java b/fe/src/main/java/com/cloudera/impala/analysis/CastExpr.java deleted file mode 100644 index 2b3b271..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/CastExpr.java +++ /dev/null @@ -1,312 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.catalog.Catalog; -import com.cloudera.impala.catalog.Db; -import com.cloudera.impala.catalog.Function; -import com.cloudera.impala.catalog.Function.CompareMode; -import com.cloudera.impala.catalog.PrimitiveType; -import com.cloudera.impala.catalog.ScalarFunction; -import com.cloudera.impala.catalog.ScalarType; -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TExpr; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TExprNodeType; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -public class CastExpr extends Expr { - // Only set for explicit casts. Null for implicit casts. - private final TypeDef targetTypeDef_; - - // True if this is a "pre-analyzed" implicit cast. - private final boolean isImplicit_; - - // True if this cast does not change the type. - private boolean noOp_ = false; - - /** - * C'tor for "pre-analyzed" implicit casts. - */ - public CastExpr(Type targetType, Expr e) { - super(); - Preconditions.checkState(targetType.isValid()); - Preconditions.checkNotNull(e); - type_ = targetType; - targetTypeDef_ = null; - isImplicit_ = true; - // replace existing implicit casts - if (e instanceof CastExpr) { - CastExpr castExpr = (CastExpr) e; - if (castExpr.isImplicit()) e = castExpr.getChild(0); - } - children_.add(e); - - // Implicit casts don't call analyze() - // TODO: this doesn't seem like the cleanest approach but there are places - // we generate these (e.g. table loading) where there is no analyzer object. - try { - analyze(); - computeNumDistinctValues(); - } catch (AnalysisException ex) { - Preconditions.checkState(false, - "Implicit casts should never throw analysis exception."); - } - isAnalyzed_ = true; - } - - /** - * C'tor for explicit casts. - */ - public CastExpr(TypeDef targetTypeDef, Expr e) { - Preconditions.checkNotNull(targetTypeDef); - Preconditions.checkNotNull(e); - isImplicit_ = false; - targetTypeDef_ = targetTypeDef; - children_.add(e); - } - - /** - * Copy c'tor used in clone(). - */ - protected CastExpr(CastExpr other) { - super(other); - targetTypeDef_ = other.targetTypeDef_; - isImplicit_ = other.isImplicit_; - noOp_ = other.noOp_; - } - - private static String getFnName(Type targetType) { - return "castTo" + targetType.getPrimitiveType().toString(); - } - - public static void initBuiltins(Db db) { - for (Type fromType : Type.getSupportedTypes()) { - if (fromType.isNull()) continue; - for (Type toType : Type.getSupportedTypes()) { - if (toType.isNull()) continue; - // Disable casting from string to boolean - if (fromType.isStringType() && toType.isBoolean()) continue; - // Disable casting from boolean/timestamp to decimal - if ((fromType.isBoolean() || fromType.isDateType()) && toType.isDecimal()) { - continue; - } - if (fromType.getPrimitiveType() == PrimitiveType.STRING - && toType.getPrimitiveType() == PrimitiveType.CHAR) { - // Allow casting from String to Char(N) - String beSymbol = "impala::CastFunctions::CastToChar"; - db.addBuiltin(ScalarFunction.createBuiltin(getFnName(ScalarType.CHAR), - Lists.newArrayList((Type) ScalarType.STRING), false, ScalarType.CHAR, - beSymbol, null, null, true)); - continue; - } - if (fromType.getPrimitiveType() == PrimitiveType.CHAR - && toType.getPrimitiveType() == PrimitiveType.CHAR) { - // Allow casting from CHAR(N) to Char(N) - String beSymbol = "impala::CastFunctions::CastToChar"; - db.addBuiltin(ScalarFunction.createBuiltin(getFnName(ScalarType.CHAR), - Lists.newArrayList((Type) ScalarType.createCharType(-1)), false, - ScalarType.CHAR, beSymbol, null, null, true)); - continue; - } - if (fromType.getPrimitiveType() == PrimitiveType.VARCHAR - && toType.getPrimitiveType() == PrimitiveType.VARCHAR) { - // Allow casting from VARCHAR(N) to VARCHAR(M) - String beSymbol = "impala::CastFunctions::CastToStringVal"; - db.addBuiltin(ScalarFunction.createBuiltin(getFnName(ScalarType.VARCHAR), - Lists.newArrayList((Type) ScalarType.VARCHAR), false, ScalarType.VARCHAR, - beSymbol, null, null, true)); - continue; - } - if (fromType.getPrimitiveType() == PrimitiveType.VARCHAR - && toType.getPrimitiveType() == PrimitiveType.CHAR) { - // Allow casting from VARCHAR(N) to CHAR(M) - String beSymbol = "impala::CastFunctions::CastToChar"; - db.addBuiltin(ScalarFunction.createBuiltin(getFnName(ScalarType.CHAR), - Lists.newArrayList((Type) ScalarType.VARCHAR), false, ScalarType.CHAR, - beSymbol, null, null, true)); - continue; - } - if (fromType.getPrimitiveType() == PrimitiveType.CHAR - && toType.getPrimitiveType() == PrimitiveType.VARCHAR) { - // Allow casting from CHAR(N) to VARCHAR(M) - String beSymbol = "impala::CastFunctions::CastToStringVal"; - db.addBuiltin(ScalarFunction.createBuiltin(getFnName(ScalarType.VARCHAR), - Lists.newArrayList((Type) ScalarType.CHAR), false, ScalarType.VARCHAR, - beSymbol, null, null, true)); - continue; - } - // Disable no-op casts - if (fromType.equals(toType) && !fromType.isDecimal()) continue; - String beClass = toType.isDecimal() || fromType.isDecimal() ? - "DecimalOperators" : "CastFunctions"; - String beSymbol = "impala::" + beClass + "::CastTo" + Function.getUdfType(toType); - db.addBuiltin(ScalarFunction.createBuiltin(getFnName(toType), - Lists.newArrayList(fromType), false, toType, beSymbol, - null, null, true)); - } - } - } - - @Override - public String toSqlImpl() { - if (isImplicit_) return getChild(0).toSql(); - return "CAST(" + getChild(0).toSql() + " AS " + targetTypeDef_.toString() + ")"; - } - - @Override - protected void treeToThriftHelper(TExpr container) { - if (noOp_) { - getChild(0).treeToThriftHelper(container); - return; - } - super.treeToThriftHelper(container); - } - - @Override - protected void toThrift(TExprNode msg) { - msg.node_type = TExprNodeType.FUNCTION_CALL; - } - - @Override - public String debugString() { - return Objects.toStringHelper(this) - .add("isImplicit", isImplicit_) - .add("target", type_) - .addValue(super.debugString()) - .toString(); - } - - public boolean isImplicit() { return isImplicit_; } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - Preconditions.checkState(!isImplicit_); - super.analyze(analyzer); - targetTypeDef_.analyze(analyzer); - type_ = targetTypeDef_.getType(); - analyze(); - } - - private void analyze() throws AnalysisException { - if (getChild(0).hasCost()) evalCost_ = getChild(0).getCost() + CAST_COST; - - Preconditions.checkNotNull(type_); - if (type_.isComplexType()) { - throw new AnalysisException( - "Unsupported cast to complex type: " + type_.toSql()); - } - - boolean readyForCharCast = - children_.get(0).getType().getPrimitiveType() == PrimitiveType.STRING || - children_.get(0).getType().getPrimitiveType() == PrimitiveType.CHAR; - if (type_.getPrimitiveType() == PrimitiveType.CHAR && !readyForCharCast) { - // Back end functions only exist to cast string types to CHAR, there is not a cast - // for every type since it is redundant with STRING. Casts to go through 2 casts: - // (1) cast to string, to stringify the value - // (2) cast to CHAR, to truncate or pad with spaces - CastExpr tostring = new CastExpr(ScalarType.STRING, children_.get(0)); - tostring.analyze(); - children_.set(0, tostring); - } - - if (children_.get(0) instanceof NumericLiteral && type_.isFloatingPointType()) { - // Special case casting a decimal literal to a floating point number. The - // decimal literal can be interpreted as either and we want to avoid casts - // since that can result in loss of accuracy. - ((NumericLiteral)children_.get(0)).explicitlyCastToFloat(type_); - } - - if (children_.get(0).getType().isNull()) { - // Make sure BE never sees TYPE_NULL - uncheckedCastChild(type_, 0); - } - - // Ensure child has non-null type (even if it's a null literal). This is required - // for the UDF interface. - if (children_.get(0) instanceof NullLiteral) { - NullLiteral nullChild = (NullLiteral)(children_.get(0)); - nullChild.uncheckedCastTo(type_); - } - - Type childType = children_.get(0).type_; - Preconditions.checkState(!childType.isNull()); - if (childType.equals(type_)) { - noOp_ = true; - return; - } - - FunctionName fnName = new FunctionName(Catalog.BUILTINS_DB, getFnName(type_)); - Type[] args = { childType }; - Function searchDesc = new Function(fnName, args, Type.INVALID, false); - if (isImplicit_) { - fn_ = Catalog.getBuiltin(searchDesc, CompareMode.IS_NONSTRICT_SUPERTYPE_OF); - Preconditions.checkState(fn_ != null); - } else { - fn_ = Catalog.getBuiltin(searchDesc, CompareMode.IS_IDENTICAL); - if (fn_ == null) { - // allow for promotion from CHAR to STRING; only if no exact match is found - fn_ = Catalog.getBuiltin(searchDesc.promoteCharsToStrings(), - CompareMode.IS_IDENTICAL); - } - } - if (fn_ == null) { - throw new AnalysisException("Invalid type cast of " + getChild(0).toSql() + - " from " + childType + " to " + type_); - } - - Preconditions.checkState(type_.matchesType(fn_.getReturnType()), - type_ + " != " + fn_.getReturnType()); - } - - /** - * Returns child expr if this expr is an implicit cast, otherwise returns 'this'. - */ - @Override - public Expr ignoreImplicitCast() { - if (isImplicit_) { - // we don't expect to see to consecutive implicit casts - Preconditions.checkState( - !(getChild(0) instanceof CastExpr) || !((CastExpr) getChild(0)).isImplicit()); - return getChild(0); - } else { - return this; - } - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj instanceof CastExpr) { - CastExpr other = (CastExpr) obj; - return isImplicit_ == other.isImplicit_ - && type_.equals(other.type_) - && super.equals(obj); - } - // Ignore implicit casts when comparing expr trees. - if (isImplicit_) return getChild(0).equals(obj); - return false; - } - - @Override - public Expr clone() { return new CastExpr(this); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/CollectionStructType.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/CollectionStructType.java b/fe/src/main/java/com/cloudera/impala/analysis/CollectionStructType.java deleted file mode 100644 index b45b856..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/CollectionStructType.java +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.ArrayList; - -import jline.internal.Preconditions; - -import com.cloudera.impala.catalog.ArrayType; -import com.cloudera.impala.catalog.MapType; -import com.cloudera.impala.catalog.ScalarType; -import com.cloudera.impala.catalog.StructField; -import com.cloudera.impala.catalog.StructType; -import com.cloudera.impala.catalog.Type; -import com.google.common.collect.Lists; - -/** - * Generated struct type describing the fields of a collection type - * that can be referenced in paths. - * - * Parent Type CollectionStructType - * array<i> --> struct<item:i,pos:bigint> - * map<k,v> --> struct<key:k,value:v> - */ -public class CollectionStructType extends StructType { - // True if this struct describes the fields of a map, - // false if it describes the fields of an array. - private final boolean isMapStruct_; - - // Field that can be skipped by implicit paths if its type is a struct. - private final StructField optionalField_; - - private CollectionStructType(ArrayList<StructField> fields, boolean isMapStruct) { - super(fields); - isMapStruct_ = isMapStruct; - if (isMapStruct_) { - optionalField_ = getField(Path.MAP_VALUE_FIELD_NAME); - } else { - optionalField_ = getField(Path.ARRAY_ITEM_FIELD_NAME); - } - Preconditions.checkNotNull(optionalField_); - } - - public static CollectionStructType createArrayStructType(ArrayType arrayType) { - Type itemType = arrayType.getItemType(); - ArrayList<StructField> fields = Lists.newArrayListWithCapacity(2); - // The item field name comes before the pos field name so that a path to the - // stored item corresponds to its physical path. - fields.add(new StructField(Path.ARRAY_ITEM_FIELD_NAME, itemType)); - fields.add(new StructField(Path.ARRAY_POS_FIELD_NAME, ScalarType.BIGINT)); - return new CollectionStructType(fields, false); - } - - public static CollectionStructType createMapStructType(MapType mapType) { - ArrayList<StructField> mapFields = Lists.newArrayListWithCapacity(2); - mapFields.add(new StructField(Path.MAP_KEY_FIELD_NAME, mapType.getKeyType())); - mapFields.add(new StructField(Path.MAP_VALUE_FIELD_NAME, mapType.getValueType())); - return new CollectionStructType(mapFields, true); - } - - public StructField getOptionalField() { return optionalField_; } - public boolean isMapStruct() { return isMapStruct_; } - public boolean isArrayStruct() { return !isMapStruct_; } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/CollectionTableRef.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/CollectionTableRef.java b/fe/src/main/java/com/cloudera/impala/analysis/CollectionTableRef.java deleted file mode 100644 index 8abed3e..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/CollectionTableRef.java +++ /dev/null @@ -1,138 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.authorization.Privilege; -import com.cloudera.impala.authorization.PrivilegeRequestBuilder; -import com.cloudera.impala.common.AnalysisException; -import com.google.common.base.Preconditions; - -/** - * Reference to a MAP or ARRAY collection type that implies its - * flattening during execution. - * TODO: We currently create a new slot in the root tuple descriptor for every - * relative collection ref, even if they have the same path. The BE currently relies on - * this behavior for setting collection slots to NULL after they have been unnested - * inside a SubplanNode. We could instead share the slot and the corresponding item tuple - * descriptor among all collection table refs with the same path. This change will - * require decoupling tuple descriptors from table aliases, i.e., a tuple descriptor - * should be able to back multiple aliases. - */ -public class CollectionTableRef extends TableRef { - ///////////////////////////////////////// - // BEGIN: Members that need to be reset() - - // Expr that returns the referenced collection. Typically a SlotRef into the - // parent scan's tuple. Result of analysis. Fully resolved against base tables. - private Expr collectionExpr_; - - // END: Members that need to be reset() - ///////////////////////////////////////// - - /** - * Create a CollectionTableRef from the original unresolved table ref as well as - * its resolved path. Sets table aliases and join-related attributes. - */ - public CollectionTableRef(TableRef tableRef, Path resolvedPath) { - super(tableRef); - Preconditions.checkState(resolvedPath.isResolved()); - resolvedPath_ = resolvedPath; - // Use the last path element as an implicit alias if no explicit alias was given. - if (hasExplicitAlias()) return; - String implicitAlias = rawPath_.get(rawPath_.size() - 1).toLowerCase(); - aliases_ = new String[] { implicitAlias }; - } - - /** - * C'tor for cloning. - */ - public CollectionTableRef(CollectionTableRef other) { - super(other); - collectionExpr_ = - (other.collectionExpr_ != null) ? other.collectionExpr_.clone() : null; - } - - /** - * Registers this collection table ref with the given analyzer and adds a slot - * descriptor for the materialized collection to be populated by parent scan. - * Also determines whether this collection table ref is correlated or not. - * - * If this function is called in the context of analyzing a WITH clause, then - * no slot is added to the parent descriptor so as to not pollute the analysis - * state of the parent block (the WITH-clause analyzer is discarded, and the - * parent analyzer could have an entirely different global state). - */ - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - desc_ = analyzer.registerTableRef(this); - if (isRelative() && !analyzer.isWithClause()) { - SlotDescriptor parentSlotDesc = analyzer.registerSlotRef(resolvedPath_); - parentSlotDesc.setItemTupleDesc(desc_); - collectionExpr_ = new SlotRef(parentSlotDesc); - // Must always be materialized to ensure the correct cardinality after unnesting. - analyzer.materializeSlots(collectionExpr_); - Analyzer parentAnalyzer = - analyzer.findAnalyzer(resolvedPath_.getRootDesc().getId()); - Preconditions.checkNotNull(parentAnalyzer); - if (parentAnalyzer != analyzer) { - TableRef parentRef = - parentAnalyzer.getTableRef(resolvedPath_.getRootDesc().getId()); - Preconditions.checkNotNull(parentRef); - // InlineViews are currently not supported as a parent ref. - Preconditions.checkState(!(parentRef instanceof InlineViewRef)); - correlatedTupleIds_.add(parentRef.getId()); - } - } - if (!isRelative()) { - // Register a table-level privilege request as well as a column-level privilege request - // for the collection-typed column. - Preconditions.checkNotNull(resolvedPath_.getRootTable()); - analyzer.registerAuthAndAuditEvent(resolvedPath_.getRootTable(), analyzer); - analyzer.registerPrivReq(new PrivilegeRequestBuilder(). - allOf(Privilege.SELECT).onColumn(desc_.getTableName().getDb(), - desc_.getTableName().getTbl(), desc_.getPath().getRawPath().get(0)) - .toRequest()); - } - isAnalyzed_ = true; - analyzeHints(analyzer); - - // TODO: For joins on nested collections some join ops can be simplified - // due to the containment relationship of the parent and child. For example, - // a FULL OUTER JOIN would become a LEFT OUTER JOIN, or a RIGHT SEMI JOIN - // would become an INNER or CROSS JOIN. - analyzeJoin(analyzer); - } - - @Override - public boolean isRelative() { - Preconditions.checkNotNull(resolvedPath_); - return resolvedPath_.getRootDesc() != null; - } - - public Expr getCollectionExpr() { return collectionExpr_; } - - @Override - protected CollectionTableRef clone() { return new CollectionTableRef(this); } - - @Override - public void reset() { - super.reset(); - collectionExpr_ = null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/ColumnDef.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/ColumnDef.java b/fe/src/main/java/com/cloudera/impala/analysis/ColumnDef.java deleted file mode 100644 index e7a3170..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/ColumnDef.java +++ /dev/null @@ -1,143 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.List; - -import org.apache.hadoop.hive.metastore.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.api.FieldSchema; - -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TColumn; -import com.cloudera.impala.util.MetaStoreUtil; -import com.google.common.base.Function; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -/** - * Represents a column definition in a CREATE/ALTER TABLE/VIEW statement. - * Column definitions in CREATE/ALTER TABLE statements require a column type, - * whereas column definitions in CREATE/ALTER VIEW statements infer the column type from - * the corresponding view definition. All column definitions have an optional comment. - * Since a column definition refers a column stored in the Metastore, the column name - * must be valid according to the Metastore's rules (see @MetaStoreUtils). - */ -public class ColumnDef { - private final String colName_; - private String comment_; - - // Required in CREATE/ALTER TABLE stmts. Set to NULL in CREATE/ALTER VIEW stmts, - // for which we setType() after analyzing the defining view definition stmt. - private final TypeDef typeDef_; - private Type type_; - - public ColumnDef(String colName, TypeDef typeDef, String comment) { - colName_ = colName.toLowerCase(); - typeDef_ = typeDef; - comment_ = comment; - } - - /** - * Creates an analyzed ColumnDef from a Hive FieldSchema. Throws if the FieldSchema's - * type is not supported. - */ - private ColumnDef(FieldSchema fs) throws AnalysisException { - Type type = Type.parseColumnType(fs.getType()); - if (type == null) { - throw new AnalysisException(String.format( - "Unsupported type '%s' in Hive field schema '%s'", - fs.getType(), fs.getName())); - } - colName_ = fs.getName(); - typeDef_ = new TypeDef(type); - comment_ = fs.getComment(); - analyze(); - } - - public void setType(Type type) { type_ = type; } - public Type getType() { return type_; } - public TypeDef getTypeDef() { return typeDef_; } - public String getColName() { return colName_; } - public void setComment(String comment) { comment_ = comment; } - public String getComment() { return comment_; } - - public void analyze() throws AnalysisException { - // Check whether the column name meets the Metastore's requirements. - if (!MetaStoreUtils.validateName(colName_)) { - throw new AnalysisException("Invalid column/field name: " + colName_); - } - if (typeDef_ != null) { - typeDef_.analyze(null); - type_ = typeDef_.getType(); - } - Preconditions.checkNotNull(type_); - Preconditions.checkState(type_.isValid()); - // Check HMS constraints of type and comment. - String typeSql = type_.toSql(); - if (typeSql.length() > MetaStoreUtil.MAX_TYPE_NAME_LENGTH) { - throw new AnalysisException(String.format( - "Type of column '%s' exceeds maximum type length of %d characters:\n" + - "%s has %d characters.", colName_, MetaStoreUtil.MAX_TYPE_NAME_LENGTH, - typeSql, typeSql.length())); - } - if (comment_ != null && - comment_.length() > MetaStoreUtil.CREATE_MAX_COMMENT_LENGTH) { - throw new AnalysisException(String.format( - "Comment of column '%s' exceeds maximum length of %d characters:\n" + - "%s has %d characters.", colName_, MetaStoreUtil.CREATE_MAX_COMMENT_LENGTH, - comment_, comment_.length())); - } - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(colName_); - if (type_ != null) { - sb.append(" " + type_.toString()); - } else { - sb.append(" " + typeDef_.toString()); - } - if (comment_ != null) sb.append(String.format(" COMMENT '%s'", comment_)); - return sb.toString(); - } - - public TColumn toThrift() { - TColumn col = new TColumn(new TColumn(getColName(), type_.toThrift())); - col.setComment(getComment()); - return col; - } - - public static List<ColumnDef> createFromFieldSchemas(List<FieldSchema> fieldSchemas) - throws AnalysisException { - List<ColumnDef> result = Lists.newArrayListWithCapacity(fieldSchemas.size()); - for (FieldSchema fs: fieldSchemas) result.add(new ColumnDef(fs)); - return result; - } - - public static List<FieldSchema> toFieldSchemas(List<ColumnDef> colDefs) { - return Lists.transform(colDefs, new Function<ColumnDef, FieldSchema>() { - public FieldSchema apply(ColumnDef colDef) { - Preconditions.checkNotNull(colDef.getType()); - return new FieldSchema(colDef.getColName(), colDef.getType().toSql(), - colDef.getComment()); - } - }); - } - -}
