http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java b/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java deleted file mode 100644 index d53aa9e..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java +++ /dev/null @@ -1,516 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.List; - -import com.cloudera.impala.authorization.Privilege; -import com.cloudera.impala.catalog.AggregateFunction; -import com.cloudera.impala.catalog.Catalog; -import com.cloudera.impala.catalog.Db; -import com.cloudera.impala.catalog.Function; -import com.cloudera.impala.catalog.ScalarFunction; -import com.cloudera.impala.catalog.ScalarType; -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.TreeNode; -import com.cloudera.impala.thrift.TAggregateExpr; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TExprNodeType; -import com.cloudera.impala.thrift.TFunctionBinaryType; -import com.google.common.base.Joiner; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -public class FunctionCallExpr extends Expr { - private final FunctionName fnName_; - private final FunctionParams params_; - private boolean isAnalyticFnCall_ = false; - private boolean isInternalFnCall_ = false; - - // Indicates whether this is a merge aggregation function that should use the merge - // instead of the update symbol. This flag also affects the behavior of - // resetAnalysisState() which is used during expr substitution. - private final boolean isMergeAggFn_; - - // Printed in toSqlImpl(), if set. Used for merge agg fns. - private String label_; - - public FunctionCallExpr(String functionName, List<Expr> params) { - this(new FunctionName(functionName), new FunctionParams(false, params)); - } - - public FunctionCallExpr(FunctionName fnName, List<Expr> params) { - this(fnName, new FunctionParams(false, params)); - } - - public FunctionCallExpr(FunctionName fnName, FunctionParams params) { - this(fnName, params, false); - } - - private FunctionCallExpr( - FunctionName fnName, FunctionParams params, boolean isMergeAggFn) { - super(); - fnName_ = fnName; - params_ = params; - isMergeAggFn_ = isMergeAggFn; - if (params.exprs() != null) children_ = Lists.newArrayList(params_.exprs()); - } - - /** - * Returns an Expr that evaluates the function call <fnName>(<params>). The returned - * Expr is not necessarily a FunctionCallExpr (example: DECODE()) - */ - public static Expr createExpr(FunctionName fnName, FunctionParams params) { - FunctionCallExpr functionCallExpr = new FunctionCallExpr(fnName, params); - if (fnName.getFnNamePath().size() == 1 - && fnName.getFnNamePath().get(0).equalsIgnoreCase("decode") - || fnName.getFnNamePath().size() == 2 - && fnName.getFnNamePath().get(0).equalsIgnoreCase(Catalog.BUILTINS_DB) - && fnName.getFnNamePath().get(1).equalsIgnoreCase("decode")) { - return new CaseExpr(functionCallExpr); - } - return functionCallExpr; - } - - /** - * Returns a new function call expr on the given params for performing the merge() - * step of the given aggregate function. - */ - public static FunctionCallExpr createMergeAggCall( - FunctionCallExpr agg, List<Expr> params) { - Preconditions.checkState(agg.isAnalyzed_); - Preconditions.checkState(agg.isAggregateFunction()); - FunctionCallExpr result = new FunctionCallExpr( - agg.fnName_, new FunctionParams(false, params), true); - // Inherit the function object from 'agg'. - result.fn_ = agg.fn_; - result.type_ = agg.type_; - // Set an explicit label based on the input agg. - if (agg.isMergeAggFn_) { - result.label_ = agg.label_; - } else { - // fn(input) becomes fn:merge(input). - result.label_ = agg.toSql().replaceFirst(agg.fnName_.toString(), - agg.fnName_.toString() + ":merge"); - } - Preconditions.checkState(!result.type_.isWildcardDecimal()); - return result; - } - - /** - * Copy c'tor used in clone(). - */ - protected FunctionCallExpr(FunctionCallExpr other) { - super(other); - fnName_ = other.fnName_; - isAnalyticFnCall_ = other.isAnalyticFnCall_; - isInternalFnCall_ = other.isInternalFnCall_; - isMergeAggFn_ = other.isMergeAggFn_; - // Clone the params in a way that keeps the children_ and the params.exprs() - // in sync. The children have already been cloned in the super c'tor. - if (other.params_.isStar()) { - Preconditions.checkState(children_.isEmpty()); - params_ = FunctionParams.createStarParam(); - } else { - params_ = new FunctionParams(other.params_.isDistinct(), children_); - } - label_ = other.label_; - } - - public boolean isMergeAggFn() { return isMergeAggFn_; } - - @Override - public void resetAnalysisState() { - isAnalyzed_ = false; - // Resolving merge agg functions after substitution may fail e.g., if the - // intermediate agg type is not the same as the output type. Preserve the original - // fn_ such that analyze() hits the special-case code for merge agg fns that - // handles this case. - if (!isMergeAggFn_) fn_ = null; - } - - @Override - public boolean equals(Object obj) { - if (!super.equals(obj)) return false; - FunctionCallExpr o = (FunctionCallExpr)obj; - return fnName_.equals(o.fnName_) && - params_.isDistinct() == o.params_.isDistinct() && - params_.isStar() == o.params_.isStar(); - } - - @Override - public String toSqlImpl() { - if (label_ != null) return label_; - // Merge agg fns should have an explicit label. - Preconditions.checkState(!isMergeAggFn_); - StringBuilder sb = new StringBuilder(); - sb.append(fnName_).append("("); - if (params_.isStar()) sb.append("*"); - if (params_.isDistinct()) sb.append("DISTINCT "); - sb.append(Joiner.on(", ").join(childrenToSql())).append(")"); - return sb.toString(); - } - - @Override - public String debugString() { - return Objects.toStringHelper(this) - .add("name", fnName_) - .add("isStar", params_.isStar()) - .add("isDistinct", params_.isDistinct()) - .addValue(super.debugString()) - .toString(); - } - - public FunctionParams getParams() { return params_; } - public boolean isScalarFunction() { - Preconditions.checkNotNull(fn_); - return fn_ instanceof ScalarFunction ; - } - - public Type getReturnType() { - Preconditions.checkNotNull(fn_); - return fn_.getReturnType(); - } - - /** - * Returns true if this is a call to a non-analytic aggregate function. - */ - public boolean isAggregateFunction() { - Preconditions.checkNotNull(fn_); - return fn_ instanceof AggregateFunction && !isAnalyticFnCall_; - } - - /** - * Returns true if this is a call to an aggregate function that returns - * non-null on an empty input (e.g. count). - */ - public boolean returnsNonNullOnEmpty() { - Preconditions.checkNotNull(fn_); - return fn_ instanceof AggregateFunction && - ((AggregateFunction)fn_).returnsNonNullOnEmpty(); - } - - public boolean isDistinct() { - Preconditions.checkState(isAggregateFunction()); - return params_.isDistinct(); - } - - public boolean ignoresDistinct() { - Preconditions.checkState(isAggregateFunction()); - return ((AggregateFunction)fn_).ignoresDistinct(); - } - - public FunctionName getFnName() { return fnName_; } - public void setIsAnalyticFnCall(boolean v) { isAnalyticFnCall_ = v; } - public void setIsInternalFnCall(boolean v) { isInternalFnCall_ = v; } - - @Override - protected void toThrift(TExprNode msg) { - if (isAggregateFunction() || isAnalyticFnCall_) { - msg.node_type = TExprNodeType.AGGREGATE_EXPR; - if (!isAnalyticFnCall_) msg.setAgg_expr(new TAggregateExpr(isMergeAggFn_)); - } else { - msg.node_type = TExprNodeType.FUNCTION_CALL; - } - } - - /** - * Aggregate functions are never constant. - */ - @Override - public boolean isConstant() { - if (fn_ != null && fn_ instanceof AggregateFunction) return false; - return super.isConstant(); - } - - // Provide better error message for some aggregate builtins. These can be - // a bit more user friendly than a generic function not found. - // TODO: should we bother to do this? We could also improve the general - // error messages. For example, listing the alternatives. - protected String getFunctionNotFoundError(Type[] argTypes) { - if (fnName_.isBuiltin()) { - // Some custom error message for builtins - if (params_.isStar()) { - return "'*' can only be used in conjunction with COUNT"; - } - if (fnName_.getFunction().equalsIgnoreCase("count")) { - if (!params_.isDistinct() && argTypes.length > 1) { - return "COUNT must have DISTINCT for multiple arguments: " + toSql(); - } - } - if (fnName_.getFunction().equalsIgnoreCase("sum")) { - return "SUM requires a numeric parameter: " + toSql(); - } - if (fnName_.getFunction().equalsIgnoreCase("avg")) { - return "AVG requires a numeric or timestamp parameter: " + toSql(); - } - } - - String[] argTypesSql = new String[argTypes.length]; - for (int i = 0; i < argTypes.length; ++i) { - argTypesSql[i] = argTypes[i].toSql(); - } - return String.format( - "No matching function with signature: %s(%s).", - fnName_, params_.isStar() ? "*" : Joiner.on(", ").join(argTypesSql)); - } - - /** - * Builtins that return decimals are specified as the wildcard decimal(decimal(*,*)) - * and the specific decimal can only be determined based on the inputs. We currently - * don't have a mechanism to specify this with the UDF interface. Until we add - * that (i.e. allowing UDFs to participate in the planning phase), we will - * manually resolve the wildcard types for the few functions that need it. - * This can only be called for functions that return wildcard decimals and the first - * argument is a wildcard decimal. - * TODO: this prevents UDFs from using wildcard decimals and is in general not scalable. - * We should add a prepare_fn() to UDFs for doing this. - */ - private Type resolveDecimalReturnType(Analyzer analyzer) throws AnalysisException { - Preconditions.checkState(type_.isWildcardDecimal()); - Preconditions.checkState(fn_.getBinaryType() == TFunctionBinaryType.BUILTIN); - Preconditions.checkState(children_.size() > 0); - - // Find first decimal input (some functions, such as if(), begin with non-decimal - // arguments). - ScalarType childType = null; - for (Expr child : children_) { - if (child.type_.isDecimal()) { - childType = (ScalarType) child.type_; - break; - } - } - Preconditions.checkState(childType != null && !childType.isWildcardDecimal()); - Type returnType = childType; - - if (fnName_.getFunction().equalsIgnoreCase("sum")) { - return childType.getMaxResolutionType(); - } - - int digitsBefore = childType.decimalPrecision() - childType.decimalScale(); - int digitsAfter = childType.decimalScale(); - if (fnName_.getFunction().equalsIgnoreCase("ceil") || - fnName_.getFunction().equalsIgnoreCase("ceiling") || - fnName_.getFunction().equals("floor") || - fnName_.getFunction().equals("dfloor")) { - // These functions just return with scale 0 but can trigger rounding. We need - // to increase the precision by 1 to handle that. - ++digitsBefore; - digitsAfter = 0; - } else if (fnName_.getFunction().equalsIgnoreCase("truncate") || - fnName_.getFunction().equalsIgnoreCase("dtrunc") || - fnName_.getFunction().equalsIgnoreCase("round") || - fnName_.getFunction().equalsIgnoreCase("dround")) { - if (children_.size() > 1) { - // The second argument to these functions is the desired scale, otherwise - // the default is 0. - Preconditions.checkState(children_.size() == 2); - if (children_.get(1).isNullLiteral()) { - throw new AnalysisException(fnName_.getFunction() + - "() cannot be called with a NULL second argument."); - } - - if (!children_.get(1).isConstant()) { - // We don't allow calling truncate or round with a non-constant second - // (desired scale) argument. e.g. select round(col1, col2). This would - // mean we don't know the scale of the resulting type and would need some - // kind of dynamic type handling which is not yet possible. This seems like - // a reasonable restriction. - throw new AnalysisException(fnName_.getFunction() + - "() must be called with a constant second argument."); - } - NumericLiteral scaleLiteral = (NumericLiteral) LiteralExpr.create( - children_.get(1), analyzer.getQueryCtx()); - digitsAfter = (int)scaleLiteral.getLongValue(); - if (Math.abs(digitsAfter) > ScalarType.MAX_SCALE) { - throw new AnalysisException("Cannot round/truncate to scales greater than " + - ScalarType.MAX_SCALE + "."); - } - // Round/Truncate to a negative scale means to round to the digit before - // the decimal e.g. round(1234.56, -2) would be 1200. - // The resulting scale is always 0. - digitsAfter = Math.max(digitsAfter, 0); - } else { - // Round()/Truncate() with no second argument. - digitsAfter = 0; - } - - if ((fnName_.getFunction().equalsIgnoreCase("round") || - fnName_.getFunction().equalsIgnoreCase("dround")) && - digitsAfter < childType.decimalScale()) { - // If we are rounding to fewer decimal places, it's possible we need another - // digit before the decimal. - ++digitsBefore; - } - } - Preconditions.checkState(returnType.isDecimal() && !returnType.isWildcardDecimal()); - return ScalarType.createDecimalTypeInternal(digitsBefore + digitsAfter, digitsAfter); - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - super.analyze(analyzer); - fnName_.analyze(analyzer); - - if (isMergeAggFn_) { - // This is the function call expr after splitting up to a merge aggregation. - // The function has already been analyzed so just do the minimal sanity - // check here. - AggregateFunction aggFn = (AggregateFunction)fn_; - Preconditions.checkNotNull(aggFn); - Type intermediateType = aggFn.getIntermediateType(); - if (intermediateType == null) intermediateType = type_; - Preconditions.checkState(!type_.isWildcardDecimal()); - return; - } - - Type[] argTypes = collectChildReturnTypes(); - - // User needs DB access. - Db db = analyzer.getDb(fnName_.getDb(), Privilege.VIEW_METADATA, true); - if (!db.containsFunction(fnName_.getFunction())) { - throw new AnalysisException(fnName_ + "() unknown"); - } - - if (fnName_.getFunction().equals("count") && params_.isDistinct()) { - // Treat COUNT(DISTINCT ...) special because of how we do the rewrite. - // There is no version of COUNT() that takes more than 1 argument but after - // the rewrite, we only need count(*). - // TODO: fix how we rewrite count distinct. - argTypes = new Type[0]; - Function searchDesc = new Function(fnName_, argTypes, Type.INVALID, false); - fn_ = db.getFunction(searchDesc, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); - type_ = fn_.getReturnType(); - // Make sure BE doesn't see any TYPE_NULL exprs - for (int i = 0; i < children_.size(); ++i) { - if (getChild(i).getType().isNull()) { - uncheckedCastChild(ScalarType.BOOLEAN, i); - } - } - return; - } - - // TODO: We allow implicit cast from string->timestamp but only - // support avg(timestamp). This means avg(string_col) would work - // from our casting rules. This is not right. - // We need to revisit where implicit casts are allowed for string - // to timestamp - if (fnName_.getFunction().equalsIgnoreCase("avg") && - children_.size() == 1 && children_.get(0).getType().isStringType()) { - throw new AnalysisException( - "AVG requires a numeric or timestamp parameter: " + toSql()); - } - - Function searchDesc = new Function(fnName_, argTypes, Type.INVALID, false); - fn_ = db.getFunction(searchDesc, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); - if (fn_ == null || (!isInternalFnCall_ && !fn_.userVisible())) { - throw new AnalysisException(getFunctionNotFoundError(argTypes)); - } - - if (isAggregateFunction()) { - // subexprs must not contain aggregates - if (TreeNode.contains(children_, Expr.isAggregatePredicate())) { - throw new AnalysisException( - "aggregate function must not contain aggregate parameters: " + this.toSql()); - } - - // .. or analytic exprs - if (Expr.contains(children_, AnalyticExpr.class)) { - throw new AnalysisException( - "aggregate function must not contain analytic parameters: " + this.toSql()); - } - - // The catalog contains count() with no arguments to handle count(*) but don't - // accept count(). - // TODO: can this be handled more cleanly. It does seem like a special case since - // no other aggregate functions (currently) can accept '*'. - if (fnName_.getFunction().equalsIgnoreCase("count") && - !params_.isStar() && children_.size() == 0) { - throw new AnalysisException("count() is not allowed."); - } - - // TODO: the distinct rewrite does not handle this but why? - if (params_.isDistinct()) { - // The second argument in group_concat(distinct) must be a constant expr that - // returns a string. - if (fnName_.getFunction().equalsIgnoreCase("group_concat") - && getChildren().size() == 2 - && !getChild(1).isConstant()) { - throw new AnalysisException("Second parameter in GROUP_CONCAT(DISTINCT)" + - " must be a constant expression that returns a string."); - } - if (fn_.getBinaryType() != TFunctionBinaryType.BUILTIN) { - throw new AnalysisException("User defined aggregates do not support DISTINCT."); - } - } - - AggregateFunction aggFn = (AggregateFunction)fn_; - if (aggFn.ignoresDistinct()) params_.setIsDistinct(false); - } - - if (params_.isIgnoreNulls() && !isAnalyticFnCall_) { - throw new AnalysisException("Function " + fnName_.getFunction().toUpperCase() - + " does not accept the keyword IGNORE NULLS."); - } - - if (isScalarFunction()) validateScalarFnParams(params_); - if (fn_ instanceof AggregateFunction - && ((AggregateFunction) fn_).isAnalyticFn() - && !((AggregateFunction) fn_).isAggregateFn() - && !isAnalyticFnCall_) { - throw new AnalysisException( - "Analytic function requires an OVER clause: " + toSql()); - } - - castForFunctionCall(false); - type_ = fn_.getReturnType(); - if (type_.isDecimal() && type_.isWildcardDecimal()) { - type_ = resolveDecimalReturnType(analyzer); - } - - // We do not allow any function to return a type CHAR or VARCHAR - // TODO add support for CHAR(N) and VARCHAR(N) return values in post 2.0, - // support for this was not added to the backend in 2.0 - if (type_.isWildcardChar() || type_.isWildcardVarchar()) { - type_ = ScalarType.STRING; - } - - // TODO(tmarshall): Differentiate based on the specific function. - if (hasChildCosts()) evalCost_ = getChildCosts() + FUNCTION_CALL_COST; - } - - /** - * Checks that no special aggregate params are included in 'params' that would be - * invalid for a scalar function. Analysis of the param exprs is not done. - */ - static void validateScalarFnParams(FunctionParams params) - throws AnalysisException { - if (params.isStar()) { - throw new AnalysisException("Cannot pass '*' to scalar function."); - } - if (params.isDistinct()) { - throw new AnalysisException("Cannot pass 'DISTINCT' to scalar function."); - } - } - - @Override - public Expr clone() { return new FunctionCallExpr(this); } -}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/FunctionName.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/FunctionName.java b/fe/src/main/java/com/cloudera/impala/analysis/FunctionName.java deleted file mode 100644 index 5609578..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/FunctionName.java +++ /dev/null @@ -1,148 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.ArrayList; - -import com.cloudera.impala.catalog.Catalog; -import com.cloudera.impala.catalog.Db; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TFunctionName; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; - -/** - * Class to represent a function name. Function names are specified as - * db.function_name. - */ -public class FunctionName { - // Only set for parsed function names. - private final ArrayList<String> fnNamePath_; - - // Set/validated during analysis. - private String db_; - private String fn_; - private boolean isBuiltin_ = false; - private boolean isAnalyzed_ = false; - - /** - * C'tor for parsed function names. The function names could be invalid. The validity - * is checked during analysis. - */ - public FunctionName(ArrayList<String> fnNamePath) { - fnNamePath_ = fnNamePath; - } - - public FunctionName(String dbName, String fn) { - db_ = (dbName != null) ? dbName.toLowerCase() : null; - fn_ = fn.toLowerCase(); - fnNamePath_ = null; - } - - public FunctionName(String fn) { - this(null, fn); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof FunctionName)) return false; - FunctionName o = (FunctionName)obj; - if ((db_ == null || o.db_ == null) && (db_ != o.db_)) { - if (db_ == null && o.db_ != null) return false; - if (db_ != null && o.db_ == null) return false; - if (!db_.equalsIgnoreCase(o.db_)) return false; - } - return fn_.equalsIgnoreCase(o.fn_); - } - - public String getDb() { return db_; } - public String getFunction() { return fn_; } - public boolean isFullyQualified() { return db_ != null; } - public boolean isBuiltin() { return isBuiltin_; } - public ArrayList<String> getFnNamePath() { return fnNamePath_; } - - @Override - public String toString() { - // The fnNamePath_ is not always set. - if (!isAnalyzed_ && fnNamePath_ != null) return Joiner.on(".").join(fnNamePath_); - if (db_ == null || isBuiltin_) return fn_; - return db_ + "." + fn_; - } - - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - analyzeFnNamePath(); - if (fn_.isEmpty()) throw new AnalysisException("Function name cannot be empty."); - for (int i = 0; i < fn_.length(); ++i) { - if (!isValidCharacter(fn_.charAt(i))) { - throw new AnalysisException( - "Function names must be all alphanumeric or underscore. " + - "Invalid name: " + fn_); - } - } - if (Character.isDigit(fn_.charAt(0))) { - throw new AnalysisException("Function cannot start with a digit: " + fn_); - } - - // Resolve the database for this function. - if (!isFullyQualified()) { - Db builtinDb = analyzer.getCatalog().getBuiltinsDb(); - if (builtinDb.containsFunction(fn_)) { - // If it isn't fully qualified and is the same name as a builtin, use - // the builtin. - db_ = Catalog.BUILTINS_DB; - isBuiltin_ = true; - } else { - db_ = analyzer.getDefaultDb(); - isBuiltin_ = false; - } - } else { - isBuiltin_ = db_.equals(Catalog.BUILTINS_DB); - } - isAnalyzed_ = true; - } - - private void analyzeFnNamePath() throws AnalysisException { - if (fnNamePath_ == null) return; - if (fnNamePath_.size() > 2 || fnNamePath_.isEmpty()) { - throw new AnalysisException( - String.format("Invalid function name: '%s'. Expected [dbname].funcname.", - Joiner.on(".").join(fnNamePath_))); - } else if (fnNamePath_.size() > 1) { - db_ = fnNamePath_.get(0); - fn_ = fnNamePath_.get(1).toLowerCase(); - } else { - Preconditions.checkState(fnNamePath_.size() == 1); - fn_ = fnNamePath_.get(0).toLowerCase(); - } - } - - private boolean isValidCharacter(char c) { - return Character.isLetterOrDigit(c) || c == '_'; - } - - public TFunctionName toThrift() { - TFunctionName name = new TFunctionName(fn_); - name.setDb_name(db_); - return name; - } - - public static FunctionName fromThrift(TFunctionName fnName) { - return new FunctionName(fnName.getDb_name(), fnName.getFunction_name()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/FunctionParams.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/FunctionParams.java b/fe/src/main/java/com/cloudera/impala/analysis/FunctionParams.java deleted file mode 100644 index e6854a0..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/FunctionParams.java +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.List; - -/** - * Return value of the grammar production that parses function - * parameters. These parameters can be for scalar or aggregate functions. - */ -class FunctionParams implements Cloneable { - private final boolean isStar_; - private boolean isDistinct_; - private boolean isIgnoreNulls_; - private final List<Expr> exprs_; - - // c'tor for non-star params - public FunctionParams(boolean isDistinct, boolean isIgnoreNulls, List<Expr> exprs) { - this.isStar_ = false; - this.isDistinct_ = isDistinct; - this.isIgnoreNulls_ = isIgnoreNulls; - this.exprs_ = exprs; - } - - // c'tor for non-star, non-ignore-nulls params - public FunctionParams(boolean isDistinct, List<Expr> exprs) { - this(isDistinct, false, exprs); - } - - // c'tor for non-star, non-distinct, non-ignore-nulls params - public FunctionParams(List<Expr> exprs) { - this(false, false, exprs); - } - - static public FunctionParams createStarParam() { - return new FunctionParams(); - } - - public boolean isStar() { return isStar_; } - public boolean isDistinct() { return isDistinct_; } - public boolean isIgnoreNulls() { return isIgnoreNulls_; } - public List<Expr> exprs() { return exprs_; } - public void setIsDistinct(boolean v) { isDistinct_ = v; } - public int size() { return exprs_ == null ? 0 : exprs_.size(); } - - // c'tor for <agg>(*) - private FunctionParams() { - exprs_ = null; - isStar_ = true; - isDistinct_ = false; - isIgnoreNulls_ = false; - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokePrivStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokePrivStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokePrivStmt.java deleted file mode 100644 index edaf22a..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokePrivStmt.java +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.List; - -import com.cloudera.impala.catalog.Role; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TGrantRevokePrivParams; -import com.cloudera.impala.thrift.TPrivilege; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; - -/** - * Represents a "GRANT/REVOKE PRIVILEGE" statement. - * All privilege checks on catalog objects are skipped when executing - * GRANT/REVOKE statements. This is because we need to be able to create - * privileges on an object before any privileges actually exist. - * The GRANT/REVOKE statement itself will be authorized (currently by - * the Sentry Service). - */ -public class GrantRevokePrivStmt extends AuthorizationStmt { - private final PrivilegeSpec privilegeSpec_; - private final String roleName_; - private final boolean isGrantPrivStmt_; - private final boolean hasGrantOpt_; - - // Set/modified during analysis - private Role role_; - - public GrantRevokePrivStmt(String roleName, PrivilegeSpec privilegeSpec, - boolean isGrantPrivStmt, boolean hasGrantOpt) { - Preconditions.checkNotNull(privilegeSpec); - Preconditions.checkNotNull(roleName); - privilegeSpec_ = privilegeSpec; - roleName_ = roleName; - isGrantPrivStmt_ = isGrantPrivStmt; - hasGrantOpt_ = hasGrantOpt; - } - - public TGrantRevokePrivParams toThrift() { - TGrantRevokePrivParams params = new TGrantRevokePrivParams(); - params.setRole_name(roleName_); - params.setIs_grant(isGrantPrivStmt_); - List<TPrivilege> privileges = privilegeSpec_.toThrift(); - for (TPrivilege privilege: privileges) { - privilege.setRole_id(role_.getId()); - privilege.setHas_grant_opt(hasGrantOpt_); - } - params.setHas_grant_opt(hasGrantOpt_); - params.setPrivileges(privileges); - return params; - } - - @Override - public String toSql() { - StringBuilder sb = new StringBuilder(isGrantPrivStmt_ ? "GRANT " : "REVOKE "); - if (!isGrantPrivStmt_ && hasGrantOpt_) sb.append("GRANT OPTION FOR "); - sb.append(privilegeSpec_.toSql()); - sb.append(isGrantPrivStmt_ ? " TO " : " FROM "); - sb.append(roleName_); - if (isGrantPrivStmt_ && hasGrantOpt_) sb.append(" WITH GRANT OPTION"); - return sb.toString(); - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - super.analyze(analyzer); - if (Strings.isNullOrEmpty(roleName_)) { - throw new AnalysisException("Role name in GRANT/REVOKE privilege cannot be " + - "empty."); - } - role_ = analyzer.getCatalog().getAuthPolicy().getRole(roleName_); - if (role_ == null) { - throw new AnalysisException(String.format("Role '%s' does not exist.", roleName_)); - } - privilegeSpec_.analyze(analyzer); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokeRoleStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokeRoleStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokeRoleStmt.java deleted file mode 100644 index 73240dc..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokeRoleStmt.java +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TGrantRevokeRoleParams; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - -/** - * Represents a "GRANT/REVOKE ROLE" statement. - */ -public class GrantRevokeRoleStmt extends AuthorizationStmt { - private final String roleName_; - private final String groupName_; - private final boolean isGrantStmt_; - - public GrantRevokeRoleStmt(String roleName, String groupName, boolean isGrantStmt) { - Preconditions.checkNotNull(roleName); - Preconditions.checkNotNull(groupName); - roleName_ = roleName; - groupName_ = groupName; - isGrantStmt_ = isGrantStmt; - } - - @Override - public String toSql() { - if (isGrantStmt_) { - return String.format("GRANT ROLE %s TO %s", roleName_, groupName_); - } else { - return String.format("REVOKE ROLE %s FROM %s", roleName_, groupName_); - } - } - - public TGrantRevokeRoleParams toThrift() { - TGrantRevokeRoleParams params = new TGrantRevokeRoleParams(); - params.setRole_names(Lists.newArrayList(roleName_)); - params.setGroup_names(Lists.newArrayList(groupName_)); - params.setIs_grant(isGrantStmt_); - return params; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - super.analyze(analyzer); - if (analyzer.getCatalog().getAuthPolicy().getRole(roleName_) == null) { - throw new AnalysisException(String.format("Role '%s' does not exist.", roleName_)); - } - if (Strings.isNullOrEmpty(roleName_)) { - throw new AnalysisException("Role name in GRANT/REVOKE ROLE cannot be empty."); - } - if (Strings.isNullOrEmpty(groupName_)) { - throw new AnalysisException("Group name in GRANT/REVOKE ROLE cannot be empty."); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/HdfsCachingOp.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/HdfsCachingOp.java b/fe/src/main/java/com/cloudera/impala/analysis/HdfsCachingOp.java deleted file mode 100644 index 1b6cff2..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/HdfsCachingOp.java +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.math.BigDecimal; - -import com.cloudera.impala.catalog.HdfsCachePool; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.THdfsCachingOp; -import com.google.common.base.Preconditions; - -/** - * Represents the partial SQL statement of specifying whether a table/partition - * should or should not be marked as cached. - */ -public class HdfsCachingOp implements ParseNode { - private final THdfsCachingOp cacheOp_; - private final BigDecimal parsedReplication_; - - /** - * Creates an HdfsCachingOp that specifies the target should be uncached - */ - public HdfsCachingOp() { - cacheOp_ = new THdfsCachingOp(false); - parsedReplication_ = null; - } - - /** - * Creates an HdfsCachingOp that specifies the target should be cached in cachePoolName - * with an optional replication factor - */ - public HdfsCachingOp(String cachePoolName, BigDecimal replication) { - cacheOp_ = new THdfsCachingOp(true); - cacheOp_.setCache_pool_name(cachePoolName); - parsedReplication_ = replication; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (cacheOp_.isSet_cached()) { - String poolName = cacheOp_.getCache_pool_name(); - Preconditions.checkNotNull(poolName); - if (poolName.isEmpty()) { - throw new AnalysisException("Cache pool name cannot be empty."); - } - - HdfsCachePool cachePool = analyzer.getCatalog().getHdfsCachePool(poolName); - if (cachePool == null) { - throw new AnalysisException( - "The specified cache pool does not exist: " + poolName); - } - - if (parsedReplication_ != null && (parsedReplication_.longValue() <= 0 || - parsedReplication_.longValue() > Short.MAX_VALUE)) { - throw new AnalysisException( - "Cache replication factor must be between 0 and Short.MAX_VALUE"); - } - - if (parsedReplication_ != null) { - cacheOp_.setReplication(parsedReplication_.shortValue()); - } - } - } - - @Override - public String toSql() { - return !shouldCache() ? "UNCACHED" : "CACHED IN '" + getCachePoolName() + "' WITH " + - "REPLICATION = " + parsedReplication_.longValue(); - } - - public THdfsCachingOp toThrift() { return cacheOp_; } - - public boolean shouldCache() { return cacheOp_.isSet_cached(); } - - public String getCachePoolName() { - return shouldCache() ? cacheOp_.getCache_pool_name() : null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/HdfsUri.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/HdfsUri.java b/fe/src/main/java/com/cloudera/impala/analysis/HdfsUri.java deleted file mode 100644 index 9fbe467..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/HdfsUri.java +++ /dev/null @@ -1,128 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; - -import com.cloudera.impala.authorization.AuthorizeableUri; -import com.cloudera.impala.authorization.Privilege; -import com.cloudera.impala.authorization.PrivilegeRequest; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.FileSystemUtil; -import com.cloudera.impala.util.FsPermissionChecker; -import com.google.common.base.Preconditions; - -/** - * Represents a Hadoop FileSystem URI in a SQL statement. - */ -public class HdfsUri { - private final String location_; - - // Set during analysis - private Path uriPath_; - - public HdfsUri(String location) { - Preconditions.checkNotNull(location); - this.location_ = location.trim(); - } - - public Path getPath() { - Preconditions.checkNotNull(uriPath_); - return uriPath_; - } - - public void analyze(Analyzer analyzer, Privilege privilege) - throws AnalysisException { - analyze(analyzer, privilege, FsAction.NONE, true); - } - - public void analyze(Analyzer analyzer, Privilege privilege, FsAction perm) - throws AnalysisException { - analyze(analyzer, privilege, perm, true); - } - - public void analyze(Analyzer analyzer, Privilege privilege, boolean registerPrivReq) - throws AnalysisException { - analyze(analyzer, privilege, FsAction.NONE, registerPrivReq); - } - - /** - * Analyzes the URI. - * Optionally check location path permission, issue warning if impala user doesn't - * have sufficient access rights. - * Optionally register a privilege request. Used by GRANT/REVOKE privilege statements. - */ - public void analyze(Analyzer analyzer, Privilege privilege, FsAction perm, - boolean registerPrivReq) throws AnalysisException { - if (location_.isEmpty()) { - throw new AnalysisException("URI path cannot be empty."); - } - - uriPath_ = new Path(location_); - if (!uriPath_.isUriPathAbsolute()) { - throw new AnalysisException("URI path must be absolute: " + uriPath_); - } - - uriPath_ = FileSystemUtil.createFullyQualifiedPath(uriPath_); - - // Check if parent path exists and if impala is allowed to access it. - Path parentPath = uriPath_.getParent(); - try { - FileSystem fs = uriPath_.getFileSystem(FileSystemUtil.getConfiguration()); - boolean pathExists = false; - StringBuilder errorMsg = new StringBuilder(); - try { - pathExists = fs.exists(parentPath); - if (!pathExists) errorMsg.append("Path does not exist."); - } catch (Exception e) { - errorMsg.append(e.getMessage()); - } - if (!pathExists) { - analyzer.addWarning(String.format("Path '%s' cannot be reached: %s", - parentPath, errorMsg.toString())); - } else if (perm != FsAction.NONE) { - FsPermissionChecker checker = FsPermissionChecker.getInstance(); - if (!checker.getPermissions(fs, parentPath).checkPermissions(perm)) { - analyzer.addWarning(String.format( - "Impala does not have %s access to path '%s'", - perm.toString(), parentPath)); - } - } - } catch (IOException e) { - throw new AnalysisException(e.getMessage(), e); - } - - if (registerPrivReq) { - analyzer.registerPrivReq(new PrivilegeRequest( - new AuthorizeableUri(uriPath_.toString()), privilege)); - } - } - - @Override - public String toString() { - // If uriPath is null (this HdfsURI has not been analyzed yet) just return the raw - // location string the caller passed in. - return uriPath_ == null ? location_ : uriPath_.toString(); - } - - public String getLocation() { return location_; } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/InPredicate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/InPredicate.java b/fe/src/main/java/com/cloudera/impala/analysis/InPredicate.java deleted file mode 100644 index 28d8f12..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/InPredicate.java +++ /dev/null @@ -1,234 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.ArrayList; -import java.util.List; - -import com.cloudera.impala.catalog.Db; -import com.cloudera.impala.catalog.Function.CompareMode; -import com.cloudera.impala.catalog.PrimitiveType; -import com.cloudera.impala.catalog.ScalarFunction; -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.Reference; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TExprNodeType; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -/** - * Class representing a [NOT] IN predicate. It determines if a specified value - * (first child) matches any value in a subquery (second child) or a list - * of values (remaining children). - */ -public class InPredicate extends Predicate { - private static final String IN_SET_LOOKUP = "in_set_lookup"; - private static final String NOT_IN_SET_LOOKUP = "not_in_set_lookup"; - private static final String IN_ITERATE= "in_iterate"; - private static final String NOT_IN_ITERATE = "not_in_iterate"; - private final boolean isNotIn_; - - public boolean isNotIn() { return isNotIn_; } - - public static void initBuiltins(Db db) { - for (Type t: Type.getSupportedTypes()) { - if (t.isNull()) continue; - // TODO we do not support codegen for CHAR and the In predicate must be codegened - // because it has variable number of arguments. This will force CHARs to be - // cast up to strings; meaning that "in" comparisons will not have CHAR comparison - // semantics. - if (t.getPrimitiveType() == PrimitiveType.CHAR) continue; - - String typeString = t.getPrimitiveType().toString().toLowerCase(); - if (t.isScalarType(PrimitiveType.VARCHAR)) typeString = "string"; - - db.addBuiltin(ScalarFunction.createBuiltin(IN_ITERATE, - Lists.newArrayList(t, t), true, Type.BOOLEAN, - "impala::InPredicate::InIterate", null, null, false)); - db.addBuiltin(ScalarFunction.createBuiltin(NOT_IN_ITERATE, - Lists.newArrayList(t, t), true, Type.BOOLEAN, - "impala::InPredicate::NotInIterate", null, null, false)); - - String prepareFn = "impala::InPredicate::SetLookupPrepare_" + typeString; - String closeFn = "impala::InPredicate::SetLookupClose_" + typeString; - - db.addBuiltin(ScalarFunction.createBuiltin(IN_SET_LOOKUP, - Lists.newArrayList(t, t), true, Type.BOOLEAN, - "impala::InPredicate::InSetLookup", prepareFn, closeFn, false)); - db.addBuiltin(ScalarFunction.createBuiltin(NOT_IN_SET_LOOKUP, - Lists.newArrayList(t, t), true, Type.BOOLEAN, - "impala::InPredicate::NotInSetLookup", prepareFn, closeFn, false)); - - } - } - - // First child is the comparison expr for which we - // should check membership in the inList (the remaining children). - public InPredicate(Expr compareExpr, List<Expr> inList, boolean isNotIn) { - children_.add(compareExpr); - children_.addAll(inList); - isNotIn_ = isNotIn; - } - - // C'tor for initializing an [NOT] IN predicate with a subquery child. - public InPredicate(Expr compareExpr, Expr subquery, boolean isNotIn) { - Preconditions.checkNotNull(compareExpr); - Preconditions.checkNotNull(subquery); - children_.add(compareExpr); - children_.add(subquery); - isNotIn_ = isNotIn; - } - - /** - * Copy c'tor used in clone(). - */ - protected InPredicate(InPredicate other) { - super(other); - isNotIn_ = other.isNotIn_; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - super.analyze(analyzer); - - if (contains(Subquery.class)) { - // An [NOT] IN predicate with a subquery must contain two children, the second of - // which is a Subquery. - if (children_.size() != 2 || !(getChild(1) instanceof Subquery)) { - throw new AnalysisException("Unsupported IN predicate with a subquery: " + - toSqlImpl()); - } - Subquery subquery = (Subquery)getChild(1); - if (!subquery.returnsScalarColumn()) { - throw new AnalysisException("Subquery must return a single column: " + - subquery.toSql()); - } - - // Ensure that the column in the lhs of the IN predicate and the result of - // the subquery are type compatible. No need to perform any - // casting at this point. Any casting needed will be performed when the - // subquery is unnested. - ArrayList<Expr> subqueryExprs = subquery.getStatement().getResultExprs(); - Expr compareExpr = children_.get(0); - Expr subqueryExpr = subqueryExprs.get(0); - analyzer.getCompatibleType(compareExpr.getType(), compareExpr, subqueryExpr); - } else { - Preconditions.checkState(getChildren().size() >= 2); - analyzer.castAllToCompatibleType(children_); - Type childType = children_.get(0).getType(); - - if (childType.isNull()) { - // Make sure the BE never sees TYPE_NULL by picking an arbitrary type - for (int i = 0; i < children_.size(); ++i) { - uncheckedCastChild(Type.BOOLEAN, i); - } - } - - // Choose SetLookup or Iterate strategy. SetLookup can be used if all the exprs in - // the IN list are constant, and is faster than iterating if the IN list is big - // enough. - boolean allConstant = true; - for (int i = 1; i < children_.size(); ++i) { - if (!children_.get(i).isConstant()) { - allConstant = false; - break; - } - } - boolean useSetLookup = allConstant; - // Threshold based on InPredicateBenchmark results - int setLookupThreshold = children_.get(0).getType().isStringType() ? 6 : 2; - if (children_.size() - 1 < setLookupThreshold) useSetLookup = false; - - // Only lookup fn_ if all subqueries have been rewritten. If the second child is a - // subquery, it will have type ArrayType, which cannot be resolved to a builtin - // function and will fail analysis. - Type[] argTypes = {getChild(0).type_, getChild(1).type_}; - if (useSetLookup) { - fn_ = getBuiltinFunction(analyzer, isNotIn_ ? NOT_IN_SET_LOOKUP : IN_SET_LOOKUP, - argTypes, CompareMode.IS_NONSTRICT_SUPERTYPE_OF); - } else { - fn_ = getBuiltinFunction(analyzer, isNotIn_ ? NOT_IN_ITERATE : IN_ITERATE, - argTypes, CompareMode.IS_NONSTRICT_SUPERTYPE_OF); - } - Preconditions.checkNotNull(fn_); - Preconditions.checkState(fn_.getReturnType().isBoolean()); - castForFunctionCall(false); - } - - // TODO: Fix selectivity_ for nested predicate - Reference<SlotRef> slotRefRef = new Reference<SlotRef>(); - Reference<Integer> idxRef = new Reference<Integer>(); - if (isSingleColumnPredicate(slotRefRef, idxRef) - && idxRef.getRef() == 0 - && slotRefRef.getRef().getNumDistinctValues() > 0) { - selectivity_ = (double) (getChildren().size() - 1) - / (double) slotRefRef.getRef().getNumDistinctValues(); - selectivity_ = Math.max(0.0, Math.min(1.0, selectivity_)); - } - - if (hasChildCosts()) { - // BINARY_PREDICATE_COST accounts for the cost of performing the comparison. - evalCost_ = getChildCosts() + BINARY_PREDICATE_COST * (children_.size() - 1); - } - } - - @Override - protected void toThrift(TExprNode msg) { - // Can't serialize a predicate with a subquery - Preconditions.checkState(!contains(Subquery.class)); - msg.node_type = TExprNodeType.FUNCTION_CALL; - } - - @Override - public String toSqlImpl() { - StringBuilder strBuilder = new StringBuilder(); - String notStr = (isNotIn_) ? "NOT " : ""; - strBuilder.append(getChild(0).toSql() + " " + notStr + "IN "); - boolean hasSubquery = contains(Subquery.class); - if (!hasSubquery) strBuilder.append("("); - for (int i = 1; i < children_.size(); ++i) { - strBuilder.append(getChild(i).toSql()); - strBuilder.append((i+1 != children_.size()) ? ", " : ""); - } - if (!hasSubquery) strBuilder.append(")"); - return strBuilder.toString(); - } - - /** - * If predicate is of the form "<SlotRef> [NOT] IN", returns the - * SlotRef. - */ - @Override - public SlotRef getBoundSlot() { - return getChild(0).unwrapSlotRef(true); - } - - /** - * Negates an InPredicate. - */ - @Override - public Expr negate() { - return new InPredicate(getChild(0), children_.subList(1, children_.size()), - !isNotIn_); - } - - @Override - public Expr clone() { return new InPredicate(this); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java b/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java deleted file mode 100644 index a6c62b0..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java +++ /dev/null @@ -1,339 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.cloudera.impala.catalog.ColumnStats; -import com.cloudera.impala.catalog.StructField; -import com.cloudera.impala.catalog.StructType; -import com.cloudera.impala.catalog.View; -import com.cloudera.impala.common.AnalysisException; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -/** - * An inline view is a query statement with an alias. Inline views can be parsed directly - * from a query string or represent a reference to a local or catalog view. - */ -public class InlineViewRef extends TableRef { - private final static Logger LOG = LoggerFactory.getLogger(SelectStmt.class); - - // Catalog or local view that is referenced. - // Null for inline views parsed directly from a query string. - private final View view_; - - // If not null, these will serve as the column labels for the inline view. This provides - // a layer of separation between column labels visible from outside the inline view - // and column labels used in the query definition. Either all or none of the column - // labels must be overridden. - private List<String> explicitColLabels_; - - ///////////////////////////////////////// - // BEGIN: Members that need to be reset() - - // The select or union statement of the inline view - protected QueryStmt queryStmt_; - - // queryStmt has its own analysis context - protected Analyzer inlineViewAnalyzer_; - - // list of tuple ids materialized by queryStmt - protected final ArrayList<TupleId> materializedTupleIds_ = Lists.newArrayList(); - - // Map inline view's output slots to the corresponding resultExpr of queryStmt. - protected final ExprSubstitutionMap smap_; - - // Map inline view's output slots to the corresponding baseTblResultExpr of queryStmt. - protected final ExprSubstitutionMap baseTblSmap_; - - // END: Members that need to be reset() - ///////////////////////////////////////// - - /** - * C'tor for creating inline views parsed directly from the a query string. - */ - public InlineViewRef(String alias, QueryStmt queryStmt) { - super(null, alias); - Preconditions.checkNotNull(queryStmt); - queryStmt_ = queryStmt; - view_ = null; - smap_ = new ExprSubstitutionMap(); - baseTblSmap_ = new ExprSubstitutionMap(); - } - - public InlineViewRef(String alias, QueryStmt queryStmt, List<String> colLabels) { - this(alias, queryStmt); - explicitColLabels_ = Lists.newArrayList(colLabels); - } - - /** - * C'tor for creating inline views that replace a local or catalog view ref. - */ - public InlineViewRef(View view, TableRef origTblRef) { - super(view.getTableName().toPath(), origTblRef.getExplicitAlias()); - queryStmt_ = view.getQueryStmt().clone(); - queryStmt_.reset(); - if (view.isLocalView()) queryStmt_.reset(); - view_ = view; - smap_ = new ExprSubstitutionMap(); - baseTblSmap_ = new ExprSubstitutionMap(); - setJoinAttrs(origTblRef); - explicitColLabels_ = view.getColLabels(); - // Set implicit aliases if no explicit one was given. - if (hasExplicitAlias()) return; - aliases_ = new String[] { - view_.getTableName().toString().toLowerCase(), view_.getName().toLowerCase() - }; - } - - /** - * C'tor for cloning. - */ - public InlineViewRef(InlineViewRef other) { - super(other); - Preconditions.checkNotNull(other.queryStmt_); - view_ = other.view_; - queryStmt_ = other.queryStmt_.clone(); - inlineViewAnalyzer_ = other.inlineViewAnalyzer_; - if (other.explicitColLabels_ != null) { - explicitColLabels_ = Lists.newArrayList(other.explicitColLabels_); - } - materializedTupleIds_.addAll(other.materializedTupleIds_); - smap_ = other.smap_.clone(); - baseTblSmap_ = other.baseTblSmap_.clone(); - } - - /** - * Analyzes the inline view query block in a child analyzer of 'analyzer', creates - * a new tuple descriptor for the inline view and registers auxiliary eq predicates - * between the slots of that descriptor and the select list exprs of the inline view; - * then performs join clause analysis. - */ - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - - // Analyze the inline view query statement with its own analyzer - inlineViewAnalyzer_ = new Analyzer(analyzer); - - // Catalog views refs require special analysis settings for authorization. - boolean isCatalogView = (view_ != null && !view_.isLocalView()); - if (isCatalogView) { - analyzer.registerAuthAndAuditEvent(view_, analyzer); - if (inlineViewAnalyzer_.isExplain()) { - // If the user does not have privileges on the view's definition - // then we report a masked authorization error so as not to reveal - // privileged information (e.g., the existence of a table). - inlineViewAnalyzer_.setAuthErrMsg( - String.format("User '%s' does not have privileges to " + - "EXPLAIN this statement.", analyzer.getUser().getName())); - } else { - // If this is not an EXPLAIN statement, auth checks for the view - // definition should be disabled. - inlineViewAnalyzer_.setEnablePrivChecks(false); - } - } - - inlineViewAnalyzer_.setUseHiveColLabels( - isCatalogView ? true : analyzer.useHiveColLabels()); - queryStmt_.analyze(inlineViewAnalyzer_); - correlatedTupleIds_.addAll(queryStmt_.getCorrelatedTupleIds(inlineViewAnalyzer_)); - if (explicitColLabels_ != null) { - Preconditions.checkState( - explicitColLabels_.size() == queryStmt_.getColLabels().size()); - } - - inlineViewAnalyzer_.setHasLimitOffsetClause( - queryStmt_.hasLimit() || queryStmt_.hasOffset()); - queryStmt_.getMaterializedTupleIds(materializedTupleIds_); - desc_ = analyzer.registerTableRef(this); - isAnalyzed_ = true; // true now that we have assigned desc - - // For constant selects we materialize its exprs into a tuple. - if (materializedTupleIds_.isEmpty()) { - Preconditions.checkState(queryStmt_ instanceof SelectStmt); - Preconditions.checkState(((SelectStmt) queryStmt_).getTableRefs().isEmpty()); - desc_.setIsMaterialized(true); - materializedTupleIds_.add(desc_.getId()); - } - - // create smap_ and baseTblSmap_ and register auxiliary eq predicates between our - // tuple descriptor's slots and our *unresolved* select list exprs; - // we create these auxiliary predicates so that the analyzer can compute the value - // transfer graph through this inline view correctly (ie, predicates can get - // propagated through the view); - // if the view stmt contains analytic functions, we cannot propagate predicates - // into the view, unless the predicates are compatible with the analytic - // function's partition by clause, because those extra filters - // would alter the results of the analytic functions (see IMPALA-1243) - // TODO: relax this a bit by allowing propagation out of the inline view (but - // not into it) - for (int i = 0; i < getColLabels().size(); ++i) { - String colName = getColLabels().get(i).toLowerCase(); - Expr colExpr = queryStmt_.getResultExprs().get(i); - Path p = new Path(desc_, Lists.newArrayList(colName)); - Preconditions.checkState(p.resolve()); - SlotDescriptor slotDesc = analyzer.registerSlotRef(p); - slotDesc.setSourceExpr(colExpr); - slotDesc.setStats(ColumnStats.fromExpr(colExpr)); - SlotRef slotRef = new SlotRef(slotDesc); - smap_.put(slotRef, colExpr); - baseTblSmap_.put(slotRef, queryStmt_.getBaseTblResultExprs().get(i)); - if (createAuxPredicate(colExpr)) { - analyzer.createAuxEquivPredicate(new SlotRef(slotDesc), colExpr.clone()); - } - } - LOG.trace("inline view " + getUniqueAlias() + " smap: " + smap_.debugString()); - LOG.trace("inline view " + getUniqueAlias() + " baseTblSmap: " + - baseTblSmap_.debugString()); - - analyzeHints(analyzer); - // Now do the remaining join analysis - analyzeJoin(analyzer); - } - - /** - * Checks if an auxiliary predicate should be created for an expr. Returns False if the - * inline view has a SELECT stmt with analytic functions and the expr is not in the - * common partition exprs of all the analytic functions computed by this inline view. - */ - public boolean createAuxPredicate(Expr e) { - if (!(queryStmt_ instanceof SelectStmt) - || !((SelectStmt) queryStmt_).hasAnalyticInfo()) { - return true; - } - AnalyticInfo analyticInfo = ((SelectStmt) queryStmt_).getAnalyticInfo(); - return analyticInfo.getCommonPartitionExprs().contains(e); - } - - /** - * Create and register a non-materialized tuple descriptor for this inline view. - * This method is called from the analyzer when registering this inline view. - * Create a non-materialized tuple descriptor for this inline view. - */ - @Override - public TupleDescriptor createTupleDescriptor(Analyzer analyzer) - throws AnalysisException { - int numColLabels = getColLabels().size(); - Preconditions.checkState(numColLabels > 0); - HashSet<String> uniqueColAliases = Sets.newHashSetWithExpectedSize(numColLabels); - ArrayList<StructField> fields = Lists.newArrayListWithCapacity(numColLabels); - for (int i = 0; i < numColLabels; ++i) { - // inline view select statement has been analyzed. Col label should be filled. - Expr selectItemExpr = queryStmt_.getResultExprs().get(i); - String colAlias = getColLabels().get(i).toLowerCase(); - - // inline view col cannot have duplicate name - if (!uniqueColAliases.add(colAlias)) { - throw new AnalysisException("duplicated inline view column alias: '" + - colAlias + "'" + " in inline view " + "'" + getUniqueAlias() + "'"); - } - fields.add(new StructField(colAlias, selectItemExpr.getType(), null)); - } - - // Create the non-materialized tuple and set its type. - TupleDescriptor result = analyzer.getDescTbl().createTupleDescriptor( - getClass().getSimpleName() + " " + getUniqueAlias()); - result.setIsMaterialized(false); - result.setType(new StructType(fields)); - return result; - } - - @Override - public List<TupleId> getMaterializedTupleIds() { - Preconditions.checkState(isAnalyzed_); - Preconditions.checkState(materializedTupleIds_.size() > 0); - return materializedTupleIds_; - } - - public Analyzer getAnalyzer() { - Preconditions.checkState(isAnalyzed_); - return inlineViewAnalyzer_; - } - - public ExprSubstitutionMap getSmap() { - Preconditions.checkState(isAnalyzed_); - return smap_; - } - - public ExprSubstitutionMap getBaseTblSmap() { - Preconditions.checkState(isAnalyzed_); - return baseTblSmap_; - } - - public QueryStmt getViewStmt() { return queryStmt_; } - public void setRewrittenViewStmt(QueryStmt stmt) { - Preconditions.checkState(getAnalyzer().containsSubquery()); - queryStmt_ = stmt; - } - - public List<String> getExplicitColLabels() { return explicitColLabels_; } - - public List<String> getColLabels() { - if (explicitColLabels_ != null) return explicitColLabels_; - return queryStmt_.getColLabels(); - } - - @Override - protected TableRef clone() { return new InlineViewRef(this); } - - @Override - public void reset() { - super.reset(); - queryStmt_.reset(); - inlineViewAnalyzer_ = null; - materializedTupleIds_.clear(); - smap_.clear(); - baseTblSmap_.clear(); - } - - @Override - protected String tableRefToSql() { - // Enclose the alias in quotes if Hive cannot parse it without quotes. - // This is needed for view compatibility between Impala and Hive. - String aliasSql = null; - String alias = getExplicitAlias(); - if (alias != null) aliasSql = ToSqlUtils.getIdentSql(alias); - if (view_ != null) { - return view_.getTableName().toSql() + (aliasSql == null ? "" : " " + aliasSql); - } - Preconditions.checkNotNull(aliasSql); - StringBuilder sql = new StringBuilder() - .append("(") - .append(queryStmt_.toSql()) - .append(") ") - .append(aliasSql); - // Add explicit col labels for debugging even though this syntax isn't supported. - if (explicitColLabels_ != null) { - sql.append(" ("); - for (int i = 0; i < getExplicitColLabels().size(); i++) { - if (i > 0) sql.append(", "); - sql.append(ToSqlUtils.getIdentSql(getExplicitColLabels().get(i))); - } - sql.append(")"); - } - return sql.toString(); - } -}
