http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/util/TSessionStateUtil.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/util/TSessionStateUtil.java b/fe/src/main/java/com/cloudera/impala/util/TSessionStateUtil.java deleted file mode 100644 index f055a58..0000000 --- a/fe/src/main/java/com/cloudera/impala/util/TSessionStateUtil.java +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.util; - -import com.cloudera.impala.thrift.TSessionState; - -/** - * Utility functions for working with TSessionState objects. - */ -public class TSessionStateUtil { - /** - * Returns the actual user to perform authorisation against for the provided - * session. That is, returns the delegated user for a session if set, otherwise - * returns the connected user. - */ - public static String getEffectiveUser(TSessionState session) { - if (session.getDelegated_user() != null && - !session.getDelegated_user().isEmpty()) { - return session.getDelegated_user(); - } - return session.getConnected_user(); - } -}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/util/UnsafeUtil.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/util/UnsafeUtil.java b/fe/src/main/java/com/cloudera/impala/util/UnsafeUtil.java deleted file mode 100644 index f0c69c5..0000000 --- a/fe/src/main/java/com/cloudera/impala/util/UnsafeUtil.java +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.util; - -import java.lang.reflect.Field; -import java.security.AccessController; -import java.security.PrivilegedAction; - -import sun.misc.Unsafe; - -@SuppressWarnings("restriction") -/** - * Utility class to access unsafe methods. - */ -public class UnsafeUtil { - // object to allow us to use unsafe APIs. This lets us read native memory without - // copies and not have to switch back and forth between little endian and big endian. - public static final Unsafe UNSAFE; - - // This is the offset to the start of the array data. (There's some bytes - // before the array data like the size and other java stuff). - private static final int BYTE_ARRAY_DATA_OFFSET; - - static { - UNSAFE = (Unsafe) AccessController.doPrivileged( - new PrivilegedAction<Object>() { - @Override - public Object run() { - try { - Field f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - return f.get(null); - } catch (NoSuchFieldException e) { - throw new Error(); - } catch (IllegalAccessException e) { - throw new Error(); - } - } - }); - - BYTE_ARRAY_DATA_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); - } - - // Copies src[srcOffset, srcOffset + len) into dst. - public static void Copy(long dst, byte[] src, int srcOffset, int len) { - UNSAFE.copyMemory(src, BYTE_ARRAY_DATA_OFFSET + srcOffset, null, dst, len); - } - - // Copies src[0, len) into dst[dstOffset, dstOffset + len). - public static void Copy(byte[] dst, int dstOffset, long src, int len) { - UNSAFE.copyMemory(null, src, dst, dstOffset + BYTE_ARRAY_DATA_OFFSET, len); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/util/Visitor.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/util/Visitor.java b/fe/src/main/java/com/cloudera/impala/util/Visitor.java deleted file mode 100644 index ac7b8e3..0000000 --- a/fe/src/main/java/com/cloudera/impala/util/Visitor.java +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.util; - -/** -* Interface for visitor pattern -*/ -public interface Visitor<T> { - - void visit(T a); - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java b/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java new file mode 100644 index 0000000..e8e9445 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java @@ -0,0 +1,742 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cloudera.impala.catalog.Type; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.common.InternalException; +import com.cloudera.impala.planner.DataPartition; +import com.cloudera.impala.thrift.TPartitionType; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +/** + * Encapsulates all the information needed to compute the aggregate functions of a single + * Select block, including a possible 2nd phase aggregation step for DISTINCT aggregate + * functions and merge aggregation steps needed for distributed execution. + * + * The latter requires a tree structure of AggregateInfo objects which express the + * original aggregate computations as well as the necessary merging aggregate + * computations. + * TODO: get rid of this by transforming + * SELECT COUNT(DISTINCT a, b, ..) GROUP BY x, y, ... + * into an equivalent query with a inline view: + * SELECT COUNT(*) FROM (SELECT DISTINCT a, b, ..., x, y, ...) GROUP BY x, y, ... + * + * The tree structure looks as follows: + * - for non-distinct aggregation: + * - aggInfo: contains the original aggregation functions and grouping exprs + * - aggInfo.mergeAggInfo: contains the merging aggregation functions (grouping + * exprs are identical) + * - for distinct aggregation (for an explanation of the phases, see + * SelectStmt.createDistinctAggInfo()): + * - aggInfo: contains the phase 1 aggregate functions and grouping exprs + * - aggInfo.2ndPhaseDistinctAggInfo: contains the phase 2 aggregate functions and + * grouping exprs + * - aggInfo.mergeAggInfo: contains the merging aggregate functions for the phase 1 + * computation (grouping exprs are identical) + * - aggInfo.2ndPhaseDistinctAggInfo.mergeAggInfo: contains the merging aggregate + * functions for the phase 2 computation (grouping exprs are identical) + * + * In general, merging aggregate computations are idempotent; in other words, + * aggInfo.mergeAggInfo == aggInfo.mergeAggInfo.mergeAggInfo. + * + * TODO: move the merge construction logic from SelectStmt into AggregateInfo + * TODO: Add query tests for aggregation with intermediate tuples with num_nodes=1. + */ +public class AggregateInfo extends AggregateInfoBase { + private final static Logger LOG = LoggerFactory.getLogger(AggregateInfo.class); + + public enum AggPhase { + FIRST, + FIRST_MERGE, + SECOND, + SECOND_MERGE; + + public boolean isMerge() { return this == FIRST_MERGE || this == SECOND_MERGE; } + }; + + // created by createMergeAggInfo() + private AggregateInfo mergeAggInfo_; + + // created by createDistinctAggInfo() + private AggregateInfo secondPhaseDistinctAggInfo_; + + private final AggPhase aggPhase_; + + // Map from all grouping and aggregate exprs to a SlotRef referencing the corresp. slot + // in the intermediate tuple. Identical to outputTupleSmap_ if no aggregateExpr has an + // output type that is different from its intermediate type. + protected ExprSubstitutionMap intermediateTupleSmap_ = new ExprSubstitutionMap(); + + // Map from all grouping and aggregate exprs to a SlotRef referencing the corresp. slot + // in the output tuple. + protected ExprSubstitutionMap outputTupleSmap_ = new ExprSubstitutionMap(); + + // Map from slots of outputTupleSmap_ to the corresponding slot in + // intermediateTupleSmap_. + protected ExprSubstitutionMap outputToIntermediateTupleSmap_ = + new ExprSubstitutionMap(); + + // if set, a subset of groupingExprs_; set and used during planning + private List<Expr> partitionExprs_; + + // C'tor creates copies of groupingExprs and aggExprs. + private AggregateInfo(ArrayList<Expr> groupingExprs, + ArrayList<FunctionCallExpr> aggExprs, AggPhase aggPhase) { + super(groupingExprs, aggExprs); + aggPhase_ = aggPhase; + } + + /** + * C'tor for cloning. + */ + private AggregateInfo(AggregateInfo other) { + super(other); + if (other.mergeAggInfo_ != null) { + mergeAggInfo_ = other.mergeAggInfo_.clone(); + } + if (other.secondPhaseDistinctAggInfo_ != null) { + secondPhaseDistinctAggInfo_ = other.secondPhaseDistinctAggInfo_.clone(); + } + aggPhase_ = other.aggPhase_; + outputTupleSmap_ = other.outputTupleSmap_.clone(); + if (other.requiresIntermediateTuple()) { + intermediateTupleSmap_ = other.intermediateTupleSmap_.clone(); + } else { + Preconditions.checkState(other.intermediateTupleDesc_ == other.outputTupleDesc_); + intermediateTupleSmap_ = outputTupleSmap_; + } + partitionExprs_ = + (other.partitionExprs_ != null) ? Expr.cloneList(other.partitionExprs_) : null; + } + + public List<Expr> getPartitionExprs() { return partitionExprs_; } + public void setPartitionExprs(List<Expr> exprs) { partitionExprs_ = exprs; } + + /** + * Creates complete AggregateInfo for groupingExprs and aggExprs, including + * aggTupleDesc and aggTupleSMap. If parameter tupleDesc != null, sets aggTupleDesc to + * that instead of creating a new descriptor (after verifying that the passed-in + * descriptor is correct for the given aggregation). + * Also creates mergeAggInfo and secondPhaseDistinctAggInfo, if needed. + * If an aggTupleDesc is created, also registers eq predicates between the + * grouping exprs and their respective slots with 'analyzer'. + */ + static public AggregateInfo create( + ArrayList<Expr> groupingExprs, ArrayList<FunctionCallExpr> aggExprs, + TupleDescriptor tupleDesc, Analyzer analyzer) + throws AnalysisException { + Preconditions.checkState( + (groupingExprs != null && !groupingExprs.isEmpty()) + || (aggExprs != null && !aggExprs.isEmpty())); + Expr.removeDuplicates(groupingExprs); + Expr.removeDuplicates(aggExprs); + AggregateInfo result = new AggregateInfo(groupingExprs, aggExprs, AggPhase.FIRST); + + // collect agg exprs with DISTINCT clause + ArrayList<FunctionCallExpr> distinctAggExprs = Lists.newArrayList(); + if (aggExprs != null) { + for (FunctionCallExpr aggExpr: aggExprs) { + if (aggExpr.isDistinct()) distinctAggExprs.add(aggExpr); + } + } + + if (distinctAggExprs.isEmpty()) { + if (tupleDesc == null) { + result.createTupleDescs(analyzer); + result.createSmaps(analyzer); + } else { + // A tupleDesc should only be given for UNION DISTINCT. + Preconditions.checkState(aggExprs == null); + result.outputTupleDesc_ = tupleDesc; + result.intermediateTupleDesc_ = tupleDesc; + } + result.createMergeAggInfo(analyzer); + } else { + // we don't allow you to pass in a descriptor for distinct aggregation + // (we need two descriptors) + Preconditions.checkState(tupleDesc == null); + result.createDistinctAggInfo(groupingExprs, distinctAggExprs, analyzer); + } + LOG.debug("agg info:\n" + result.debugString()); + return result; + } + + /** + * Create aggregate info for select block containing aggregate exprs with + * DISTINCT clause. + * This creates: + * - aggTupleDesc + * - a complete secondPhaseDistinctAggInfo + * - mergeAggInfo + * + * At the moment, we require that all distinct aggregate + * functions be applied to the same set of exprs (ie, we can't do something + * like SELECT COUNT(DISTINCT id), COUNT(DISTINCT address)). + * Aggregation happens in two successive phases: + * - the first phase aggregates by all grouping exprs plus all parameter exprs + * of DISTINCT aggregate functions + * + * Example: + * SELECT a, COUNT(DISTINCT b, c), MIN(d), COUNT(*) FROM T GROUP BY a + * - 1st phase grouping exprs: a, b, c + * - 1st phase agg exprs: MIN(d), COUNT(*) + * - 2nd phase grouping exprs: a + * - 2nd phase agg exprs: COUNT(*), MIN(<MIN(d) from 1st phase>), + * SUM(<COUNT(*) from 1st phase>) + * + * TODO: expand implementation to cover the general case; this will require + * a different execution strategy + */ + private void createDistinctAggInfo( + ArrayList<Expr> origGroupingExprs, + ArrayList<FunctionCallExpr> distinctAggExprs, Analyzer analyzer) + throws AnalysisException { + Preconditions.checkState(!distinctAggExprs.isEmpty()); + // make sure that all DISTINCT params are the same; + // ignore top-level implicit casts in the comparison, we might have inserted + // those during analysis + ArrayList<Expr> expr0Children = Lists.newArrayList(); + + if (distinctAggExprs.get(0).getFnName().getFunction().equalsIgnoreCase( + "group_concat")) { + // Ignore separator parameter, otherwise the same would have to be present for all + // other distinct aggregates as well. + // TODO: Deal with constant exprs more generally, instead of special-casing + // group_concat(). + expr0Children.add(distinctAggExprs.get(0).getChild(0).ignoreImplicitCast()); + } else { + for (Expr expr : distinctAggExprs.get(0).getChildren()) { + expr0Children.add(expr.ignoreImplicitCast()); + } + } + for (int i = 1; i < distinctAggExprs.size(); ++i) { + ArrayList<Expr> exprIChildren = Lists.newArrayList(); + if (distinctAggExprs.get(i).getFnName().getFunction().equalsIgnoreCase( + "group_concat")) { + exprIChildren.add(distinctAggExprs.get(i).getChild(0).ignoreImplicitCast()); + } else { + for (Expr expr : distinctAggExprs.get(i).getChildren()) { + exprIChildren.add(expr.ignoreImplicitCast()); + } + } + if (!Expr.equalLists(expr0Children, exprIChildren)) { + throw new AnalysisException( + "all DISTINCT aggregate functions need to have the same set of " + + "parameters as " + distinctAggExprs.get(0).toSql() + + "; deviating function: " + distinctAggExprs.get(i).toSql()); + } + } + + // add DISTINCT parameters to grouping exprs + groupingExprs_.addAll(expr0Children); + + // remove DISTINCT aggregate functions from aggExprs + aggregateExprs_.removeAll(distinctAggExprs); + + createTupleDescs(analyzer); + createSmaps(analyzer); + createMergeAggInfo(analyzer); + createSecondPhaseAggInfo(origGroupingExprs, distinctAggExprs, analyzer); + } + + public AggregateInfo getMergeAggInfo() { return mergeAggInfo_; } + public AggregateInfo getSecondPhaseDistinctAggInfo() { + return secondPhaseDistinctAggInfo_; + } + public AggPhase getAggPhase() { return aggPhase_; } + public boolean isMerge() { return aggPhase_.isMerge(); } + public boolean isDistinctAgg() { return secondPhaseDistinctAggInfo_ != null; } + public ExprSubstitutionMap getIntermediateSmap() { return intermediateTupleSmap_; } + public ExprSubstitutionMap getOutputSmap() { return outputTupleSmap_; } + public ExprSubstitutionMap getOutputToIntermediateSmap() { + return outputToIntermediateTupleSmap_; + } + + public boolean hasAggregateExprs() { + return !aggregateExprs_.isEmpty() || + (secondPhaseDistinctAggInfo_ != null && + !secondPhaseDistinctAggInfo_.getAggregateExprs().isEmpty()); + } + + /** + * Return the tuple id produced in the final aggregation step. + */ + public TupleId getResultTupleId() { + if (isDistinctAgg()) return secondPhaseDistinctAggInfo_.getOutputTupleId(); + return getOutputTupleId(); + } + + public ArrayList<FunctionCallExpr> getMaterializedAggregateExprs() { + ArrayList<FunctionCallExpr> result = Lists.newArrayList(); + for (Integer i: materializedSlots_) { + result.add(aggregateExprs_.get(i)); + } + return result; + } + + /** + * Append ids of all slots that are being referenced in the process + * of performing the aggregate computation described by this AggregateInfo. + */ + public void getRefdSlots(List<SlotId> ids) { + Preconditions.checkState(outputTupleDesc_ != null); + if (groupingExprs_ != null) { + Expr.getIds(groupingExprs_, null, ids); + } + Expr.getIds(aggregateExprs_, null, ids); + // The backend assumes that the entire aggTupleDesc is materialized + for (int i = 0; i < outputTupleDesc_.getSlots().size(); ++i) { + ids.add(outputTupleDesc_.getSlots().get(i).getId()); + } + } + + /** + * Substitute all the expressions (grouping expr, aggregate expr) and update our + * substitution map according to the given substitution map: + * - smap typically maps from tuple t1 to tuple t2 (example: the smap of an + * inline view maps the virtual table ref t1 into a base table ref t2) + * - our grouping and aggregate exprs need to be substituted with the given + * smap so that they also reference t2 + * - aggTupleSMap needs to be recomputed to map exprs based on t2 + * onto our aggTupleDesc (ie, the left-hand side needs to be substituted with + * smap) + * - mergeAggInfo: this is not affected, because + * * its grouping and aggregate exprs only reference aggTupleDesc_ + * * its smap is identical to aggTupleSMap_ + * - 2ndPhaseDistinctAggInfo: + * * its grouping and aggregate exprs also only reference aggTupleDesc_ + * and are therefore not affected + * * its smap needs to be recomputed to map exprs based on t2 to its own + * aggTupleDesc + */ + public void substitute(ExprSubstitutionMap smap, Analyzer analyzer) + throws InternalException { + groupingExprs_ = Expr.substituteList(groupingExprs_, smap, analyzer, false); + LOG.trace("AggInfo: grouping_exprs=" + Expr.debugString(groupingExprs_)); + + // The smap in this case should not substitute the aggs themselves, only + // their subexpressions. + List<Expr> substitutedAggs = + Expr.substituteList(aggregateExprs_, smap, analyzer, false); + aggregateExprs_.clear(); + for (Expr substitutedAgg: substitutedAggs) { + aggregateExprs_.add((FunctionCallExpr) substitutedAgg); + } + + LOG.trace("AggInfo: agg_exprs=" + Expr.debugString(aggregateExprs_)); + outputTupleSmap_.substituteLhs(smap, analyzer); + intermediateTupleSmap_.substituteLhs(smap, analyzer); + if (secondPhaseDistinctAggInfo_ != null) { + secondPhaseDistinctAggInfo_.substitute(smap, analyzer); + } + } + + /** + * Create the info for an aggregation node that merges its pre-aggregated inputs: + * - pre-aggregation is computed by 'this' + * - tuple desc and smap are the same as that of the input (we're materializing + * the same logical tuple) + * - grouping exprs: slotrefs to the input's grouping slots + * - aggregate exprs: aggregation of the input's aggregateExprs slots + * + * The returned AggregateInfo shares its descriptor and smap with the input info; + * createAggTupleDesc() must not be called on it. + */ + private void createMergeAggInfo(Analyzer analyzer) { + Preconditions.checkState(mergeAggInfo_ == null); + TupleDescriptor inputDesc = intermediateTupleDesc_; + // construct grouping exprs + ArrayList<Expr> groupingExprs = Lists.newArrayList(); + for (int i = 0; i < getGroupingExprs().size(); ++i) { + SlotRef slotRef = new SlotRef(inputDesc.getSlots().get(i)); + groupingExprs.add(slotRef); + } + + // construct agg exprs + ArrayList<FunctionCallExpr> aggExprs = Lists.newArrayList(); + for (int i = 0; i < getAggregateExprs().size(); ++i) { + FunctionCallExpr inputExpr = getAggregateExprs().get(i); + Preconditions.checkState(inputExpr.isAggregateFunction()); + Expr aggExprParam = + new SlotRef(inputDesc.getSlots().get(i + getGroupingExprs().size())); + FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall( + inputExpr, Lists.newArrayList(aggExprParam)); + aggExpr.analyzeNoThrow(analyzer); + aggExprs.add(aggExpr); + } + + AggPhase aggPhase = + (aggPhase_ == AggPhase.FIRST) ? AggPhase.FIRST_MERGE : AggPhase.SECOND_MERGE; + mergeAggInfo_ = new AggregateInfo(groupingExprs, aggExprs, aggPhase); + mergeAggInfo_.intermediateTupleDesc_ = intermediateTupleDesc_; + mergeAggInfo_.outputTupleDesc_ = outputTupleDesc_; + mergeAggInfo_.intermediateTupleSmap_ = intermediateTupleSmap_; + mergeAggInfo_.outputTupleSmap_ = outputTupleSmap_; + mergeAggInfo_.materializedSlots_ = materializedSlots_; + } + + /** + * Creates an IF function call that returns NULL if any of the slots + * at indexes [firstIdx, lastIdx] return NULL. + * For example, the resulting IF function would like this for 3 slots: + * IF(IsNull(slot1), NULL, IF(IsNull(slot2), NULL, slot3)) + * Returns null if firstIdx is greater than lastIdx. + * Returns a SlotRef to the last slot if there is only one slot in range. + */ + private Expr createCountDistinctAggExprParam(int firstIdx, int lastIdx, + ArrayList<SlotDescriptor> slots) { + if (firstIdx > lastIdx) return null; + + Expr elseExpr = new SlotRef(slots.get(lastIdx)); + if (firstIdx == lastIdx) return elseExpr; + + for (int i = lastIdx - 1; i >= firstIdx; --i) { + ArrayList<Expr> ifArgs = Lists.newArrayList(); + SlotRef slotRef = new SlotRef(slots.get(i)); + // Build expr: IF(IsNull(slotRef), NULL, elseExpr) + Expr isNullPred = new IsNullPredicate(slotRef, false); + ifArgs.add(isNullPred); + ifArgs.add(new NullLiteral()); + ifArgs.add(elseExpr); + elseExpr = new FunctionCallExpr("if", ifArgs); + } + return elseExpr; + } + + /** + * Create the info for an aggregation node that computes the second phase of + * DISTINCT aggregate functions. + * (Refer to createDistinctAggInfo() for an explanation of the phases.) + * - 'this' is the phase 1 aggregation + * - grouping exprs are those of the original query (param origGroupingExprs) + * - aggregate exprs for the DISTINCT agg fns: these are aggregating the grouping + * slots that were added to the original grouping slots in phase 1; + * count is mapped to count(*) and sum is mapped to sum + * - other aggregate exprs: same as the non-DISTINCT merge case + * (count is mapped to sum, everything else stays the same) + * + * This call also creates the tuple descriptor and smap for the returned AggregateInfo. + */ + private void createSecondPhaseAggInfo( + ArrayList<Expr> origGroupingExprs, + ArrayList<FunctionCallExpr> distinctAggExprs, Analyzer analyzer) + throws AnalysisException { + Preconditions.checkState(secondPhaseDistinctAggInfo_ == null); + Preconditions.checkState(!distinctAggExprs.isEmpty()); + // The output of the 1st phase agg is the 1st phase intermediate. + TupleDescriptor inputDesc = intermediateTupleDesc_; + + // construct agg exprs for original DISTINCT aggregate functions + // (these aren't part of aggExprs_) + ArrayList<FunctionCallExpr> secondPhaseAggExprs = Lists.newArrayList(); + for (FunctionCallExpr inputExpr: distinctAggExprs) { + Preconditions.checkState(inputExpr.isAggregateFunction()); + FunctionCallExpr aggExpr = null; + if (inputExpr.getFnName().getFunction().equals("count")) { + // COUNT(DISTINCT ...) -> + // COUNT(IF(IsNull(<agg slot 1>), NULL, IF(IsNull(<agg slot 2>), NULL, ...))) + // We need the nested IF to make sure that we do not count + // column-value combinations if any of the distinct columns are NULL. + // This behavior is consistent with MySQL. + Expr ifExpr = createCountDistinctAggExprParam(origGroupingExprs.size(), + origGroupingExprs.size() + inputExpr.getChildren().size() - 1, + inputDesc.getSlots()); + Preconditions.checkNotNull(ifExpr); + ifExpr.analyzeNoThrow(analyzer); + aggExpr = new FunctionCallExpr("count", Lists.newArrayList(ifExpr)); + } else if (inputExpr.getFnName().getFunction().equals("group_concat")) { + // Syntax: GROUP_CONCAT([DISTINCT] expression [, separator]) + ArrayList<Expr> exprList = Lists.newArrayList(); + // Add "expression" parameter. Need to get it from the inputDesc's slots so the + // tuple reference is correct. + exprList.add(new SlotRef(inputDesc.getSlots().get(origGroupingExprs.size()))); + // Check if user provided a custom separator + if (inputExpr.getChildren().size() == 2) exprList.add(inputExpr.getChild(1)); + aggExpr = new FunctionCallExpr(inputExpr.getFnName(), exprList); + } else { + // SUM(DISTINCT <expr>) -> SUM(<last grouping slot>); + // (MIN(DISTINCT ...) and MAX(DISTINCT ...) have their DISTINCT turned + // off during analysis, and AVG() is changed to SUM()/COUNT()) + Expr aggExprParam = + new SlotRef(inputDesc.getSlots().get(origGroupingExprs.size())); + aggExpr = new FunctionCallExpr(inputExpr.getFnName(), + Lists.newArrayList(aggExprParam)); + } + secondPhaseAggExprs.add(aggExpr); + } + + // map all the remaining agg fns + for (int i = 0; i < aggregateExprs_.size(); ++i) { + FunctionCallExpr inputExpr = aggregateExprs_.get(i); + Preconditions.checkState(inputExpr.isAggregateFunction()); + // we're aggregating an intermediate slot of the 1st agg phase + Expr aggExprParam = + new SlotRef(inputDesc.getSlots().get(i + getGroupingExprs().size())); + FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall( + inputExpr, Lists.newArrayList(aggExprParam)); + secondPhaseAggExprs.add(aggExpr); + } + Preconditions.checkState( + secondPhaseAggExprs.size() == aggregateExprs_.size() + distinctAggExprs.size()); + + for (FunctionCallExpr aggExpr: secondPhaseAggExprs) { + aggExpr.analyzeNoThrow(analyzer); + Preconditions.checkState(aggExpr.isAggregateFunction()); + } + + ArrayList<Expr> substGroupingExprs = + Expr.substituteList(origGroupingExprs, intermediateTupleSmap_, analyzer, false); + secondPhaseDistinctAggInfo_ = + new AggregateInfo(substGroupingExprs, secondPhaseAggExprs, AggPhase.SECOND); + secondPhaseDistinctAggInfo_.createTupleDescs(analyzer); + secondPhaseDistinctAggInfo_.createSecondPhaseAggSMap(this, distinctAggExprs); + secondPhaseDistinctAggInfo_.createMergeAggInfo(analyzer); + } + + /** + * Create smap to map original grouping and aggregate exprs onto output + * of secondPhaseDistinctAggInfo. + */ + private void createSecondPhaseAggSMap( + AggregateInfo inputAggInfo, ArrayList<FunctionCallExpr> distinctAggExprs) { + outputTupleSmap_.clear(); + int slotIdx = 0; + ArrayList<SlotDescriptor> slotDescs = outputTupleDesc_.getSlots(); + + int numDistinctParams = distinctAggExprs.get(0).getChildren().size(); + // If we are counting distinct params of group_concat, we cannot include the custom + // separator since it is not a distinct param. + if (distinctAggExprs.get(0).getFnName().getFunction().equalsIgnoreCase( + "group_concat") + && numDistinctParams == 2) { + --numDistinctParams; + } + int numOrigGroupingExprs = + inputAggInfo.getGroupingExprs().size() - numDistinctParams; + Preconditions.checkState(slotDescs.size() == + numOrigGroupingExprs + distinctAggExprs.size() + + inputAggInfo.getAggregateExprs().size()); + + // original grouping exprs -> first m slots + for (int i = 0; i < numOrigGroupingExprs; ++i, ++slotIdx) { + Expr groupingExpr = inputAggInfo.getGroupingExprs().get(i); + outputTupleSmap_.put( + groupingExpr.clone(), new SlotRef(slotDescs.get(slotIdx))); + } + + // distinct agg exprs -> next n slots + for (int i = 0; i < distinctAggExprs.size(); ++i, ++slotIdx) { + Expr aggExpr = distinctAggExprs.get(i); + outputTupleSmap_.put( + aggExpr.clone(), (new SlotRef(slotDescs.get(slotIdx)))); + } + + // remaining agg exprs -> remaining slots + for (int i = 0; i < inputAggInfo.getAggregateExprs().size(); ++i, ++slotIdx) { + Expr aggExpr = inputAggInfo.getAggregateExprs().get(i); + outputTupleSmap_.put(aggExpr.clone(), new SlotRef(slotDescs.get(slotIdx))); + } + } + + /** + * Populates the output and intermediate smaps based on the output and intermediate + * tuples that are assumed to be set. If an intermediate tuple is required, also + * populates the output-to-intermediate smap and registers auxiliary equivalence + * predicates between the grouping slots of the two tuples. + */ + public void createSmaps(Analyzer analyzer) { + Preconditions.checkNotNull(outputTupleDesc_); + Preconditions.checkNotNull(intermediateTupleDesc_); + + List<Expr> exprs = Lists.newArrayListWithCapacity( + groupingExprs_.size() + aggregateExprs_.size()); + exprs.addAll(groupingExprs_); + exprs.addAll(aggregateExprs_); + for (int i = 0; i < exprs.size(); ++i) { + outputTupleSmap_.put(exprs.get(i).clone(), + new SlotRef(outputTupleDesc_.getSlots().get(i))); + if (!requiresIntermediateTuple()) continue; + intermediateTupleSmap_.put(exprs.get(i).clone(), + new SlotRef(intermediateTupleDesc_.getSlots().get(i))); + outputToIntermediateTupleSmap_.put( + new SlotRef(outputTupleDesc_.getSlots().get(i)), + new SlotRef(intermediateTupleDesc_.getSlots().get(i))); + if (i < groupingExprs_.size()) { + analyzer.createAuxEquivPredicate( + new SlotRef(outputTupleDesc_.getSlots().get(i)), + new SlotRef(intermediateTupleDesc_.getSlots().get(i))); + } + } + if (!requiresIntermediateTuple()) intermediateTupleSmap_ = outputTupleSmap_; + + LOG.trace("output smap=" + outputTupleSmap_.debugString()); + LOG.trace("intermediate smap=" + intermediateTupleSmap_.debugString()); + } + + /** + * Mark slots required for this aggregation as materialized: + * - all grouping output slots as well as grouping exprs + * - for non-distinct aggregation: the aggregate exprs of materialized aggregate slots; + * this assumes that the output slots corresponding to aggregate exprs have already + * been marked by the consumer of this select block + * - for distinct aggregation, we mark all aggregate output slots in order to keep + * things simple + * Also computes materializedAggregateExprs. + * This call must be idempotent because it may be called more than once for Union stmt. + */ + @Override + public void materializeRequiredSlots(Analyzer analyzer, ExprSubstitutionMap smap) { + for (int i = 0; i < groupingExprs_.size(); ++i) { + outputTupleDesc_.getSlots().get(i).setIsMaterialized(true); + intermediateTupleDesc_.getSlots().get(i).setIsMaterialized(true); + } + + // collect input exprs: grouping exprs plus aggregate exprs that need to be + // materialized + materializedSlots_.clear(); + List<Expr> exprs = Lists.newArrayList(); + exprs.addAll(groupingExprs_); + for (int i = 0; i < aggregateExprs_.size(); ++i) { + SlotDescriptor slotDesc = + outputTupleDesc_.getSlots().get(groupingExprs_.size() + i); + SlotDescriptor intermediateSlotDesc = + intermediateTupleDesc_.getSlots().get(groupingExprs_.size() + i); + if (isDistinctAgg()) { + slotDesc.setIsMaterialized(true); + intermediateSlotDesc.setIsMaterialized(true); + } + if (!slotDesc.isMaterialized()) continue; + intermediateSlotDesc.setIsMaterialized(true); + exprs.add(aggregateExprs_.get(i)); + materializedSlots_.add(i); + } + List<Expr> resolvedExprs = Expr.substituteList(exprs, smap, analyzer, false); + analyzer.materializeSlots(resolvedExprs); + + if (isDistinctAgg()) { + secondPhaseDistinctAggInfo_.materializeRequiredSlots(analyzer, null); + } + } + + /** + * Checks if all materialized aggregate expressions have distinct semantics. + * It returns true if either of the following is true: + * (1) all materialized aggregate expressions have distinct semantics + * (e.g. MIN, MAX, NDV). In other words, this optimization will work + * for COUNT(DISTINCT c) but not COUNT(c). + * (2) there are no aggregate expressions but only grouping expressions. + */ + public boolean hasAllDistinctAgg() { + if (hasAggregateExprs()) { + for (FunctionCallExpr aggExpr : getMaterializedAggregateExprs()) { + if (!aggExpr.isDistinct() && !aggExpr.ignoresDistinct()) return false; + } + } else { + Preconditions.checkState(!groupingExprs_.isEmpty()); + } + return true; + } + + /** + * Validates the internal state of this agg info: Checks that the number of + * materialized slots of the output tuple corresponds to the number of materialized + * aggregate functions plus the number of grouping exprs. Also checks that the return + * types of the aggregate and grouping exprs correspond to the slots in the output + * tuple. + */ + public void checkConsistency() { + ArrayList<SlotDescriptor> slots = outputTupleDesc_.getSlots(); + + // Check materialized slots. + int numMaterializedSlots = 0; + for (SlotDescriptor slotDesc: slots) { + if (slotDesc.isMaterialized()) ++numMaterializedSlots; + } + Preconditions.checkState(numMaterializedSlots == + materializedSlots_.size() + groupingExprs_.size()); + + // Check that grouping expr return types match the slot descriptors. + int slotIdx = 0; + for (int i = 0; i < groupingExprs_.size(); ++i) { + Expr groupingExpr = groupingExprs_.get(i); + Type slotType = slots.get(slotIdx).getType(); + Preconditions.checkState(groupingExpr.getType().equals(slotType), + String.format("Grouping expr %s returns type %s but its output tuple " + + "slot has type %s", groupingExpr.toSql(), + groupingExpr.getType().toString(), slotType.toString())); + ++slotIdx; + } + // Check that aggregate expr return types match the slot descriptors. + for (int i = 0; i < aggregateExprs_.size(); ++i) { + Expr aggExpr = aggregateExprs_.get(i); + Type slotType = slots.get(slotIdx).getType(); + Preconditions.checkState(aggExpr.getType().equals(slotType), + String.format("Agg expr %s returns type %s but its output tuple " + + "slot has type %s", aggExpr.toSql(), aggExpr.getType().toString(), + slotType.toString())); + ++slotIdx; + } + } + + /** + * Returns DataPartition derived from grouping exprs. + * Returns unpartitioned spec if no grouping. + * TODO: this won't work when we start supporting range partitions, + * because we could derive both hash and order-based partitions + */ + public DataPartition getPartition() { + if (groupingExprs_.isEmpty()) { + return DataPartition.UNPARTITIONED; + } else { + return DataPartition.hashPartitioned(groupingExprs_); + } + } + + @Override + public String debugString() { + StringBuilder out = new StringBuilder(super.debugString()); + out.append(Objects.toStringHelper(this) + .add("phase", aggPhase_) + .add("intermediate_smap", intermediateTupleSmap_.debugString()) + .add("output_smap", outputTupleSmap_.debugString()) + .toString()); + if (mergeAggInfo_ != this && mergeAggInfo_ != null) { + out.append("\nmergeAggInfo:\n" + mergeAggInfo_.debugString()); + } + if (secondPhaseDistinctAggInfo_ != null) { + out.append("\nsecondPhaseDistinctAggInfo:\n" + + secondPhaseDistinctAggInfo_.debugString()); + } + return out.toString(); + } + + @Override + protected String tupleDebugName() { return "agg-tuple"; } + + @Override + public AggregateInfo clone() { return new AggregateInfo(this); } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/AggregateInfoBase.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AggregateInfoBase.java b/fe/src/main/java/org/apache/impala/analysis/AggregateInfoBase.java new file mode 100644 index 0000000..f3ad3f8 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/AggregateInfoBase.java @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cloudera.impala.catalog.AggregateFunction; +import com.cloudera.impala.catalog.ColumnStats; +import com.cloudera.impala.catalog.Type; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +/** + * Base class for AggregateInfo and AnalyticInfo containing the intermediate and output + * tuple descriptors as well as their smaps for evaluating aggregate functions. + */ +public abstract class AggregateInfoBase { + private final static Logger LOG = + LoggerFactory.getLogger(AggregateInfoBase.class); + + // For aggregations: All unique grouping expressions from a select block. + // For analytics: Empty. + protected ArrayList<Expr> groupingExprs_; + + // For aggregations: All unique aggregate expressions from a select block. + // For analytics: The results of AnalyticExpr.getFnCall() for the unique + // AnalyticExprs of a select block. + protected ArrayList<FunctionCallExpr> aggregateExprs_; + + // The tuple into which the intermediate output of an aggregation is materialized. + // Contains groupingExprs.size() + aggregateExprs.size() slots, the first of which + // contain the values of the grouping exprs, followed by slots into which the + // aggregateExprs' update()/merge() symbols materialize their output, i.e., slots + // of the aggregate functions' intermediate types. + // Identical to outputTupleDesc_ if no aggregateExpr has an output type that is + // different from its intermediate type. + protected TupleDescriptor intermediateTupleDesc_; + + // The tuple into which the final output of the aggregation is materialized. + // Contains groupingExprs.size() + aggregateExprs.size() slots, the first of which + // contain the values of the grouping exprs, followed by slots into which the + // aggregateExprs' finalize() symbol write its result, i.e., slots of the aggregate + // functions' output types. + protected TupleDescriptor outputTupleDesc_; + + // For aggregation: indices into aggregate exprs for that need to be materialized + // For analytics: indices into the analytic exprs and their corresponding aggregate + // exprs that need to be materialized. + // Populated in materializeRequiredSlots() which must be implemented by subclasses. + protected ArrayList<Integer> materializedSlots_ = Lists.newArrayList(); + + protected AggregateInfoBase(ArrayList<Expr> groupingExprs, + ArrayList<FunctionCallExpr> aggExprs) { + Preconditions.checkState(groupingExprs != null || aggExprs != null); + groupingExprs_ = + groupingExprs != null ? Expr.cloneList(groupingExprs) : new ArrayList<Expr>(); + Preconditions.checkState(aggExprs != null || !(this instanceof AnalyticInfo)); + aggregateExprs_ = + aggExprs != null ? Expr.cloneList(aggExprs) : new ArrayList<FunctionCallExpr>(); + } + + /** + * C'tor for cloning. + */ + protected AggregateInfoBase(AggregateInfoBase other) { + groupingExprs_ = + (other.groupingExprs_ != null) ? Expr.cloneList(other.groupingExprs_) : null; + aggregateExprs_ = + (other.aggregateExprs_ != null) ? Expr.cloneList(other.aggregateExprs_) : null; + intermediateTupleDesc_ = other.intermediateTupleDesc_; + outputTupleDesc_ = other.outputTupleDesc_; + materializedSlots_ = Lists.newArrayList(other.materializedSlots_); + } + + /** + * Creates the intermediate and output tuple descriptors. If no agg expr has an + * intermediate type different from its output type, then only the output tuple + * descriptor is created and the intermediate tuple is set to the output tuple. + */ + protected void createTupleDescs(Analyzer analyzer) { + // Create the intermediate tuple desc first, so that the tuple ids are increasing + // from bottom to top in the plan tree. + intermediateTupleDesc_ = createTupleDesc(analyzer, false); + if (requiresIntermediateTuple(aggregateExprs_)) { + outputTupleDesc_ = createTupleDesc(analyzer, true); + } else { + outputTupleDesc_ = intermediateTupleDesc_; + } + } + + /** + * Returns a tuple descriptor for the aggregation/analytic's intermediate or final + * result, depending on whether isOutputTuple is true or false. + * Also updates the appropriate substitution map, and creates and registers auxiliary + * equality predicates between the grouping slots and the grouping exprs. + */ + private TupleDescriptor createTupleDesc(Analyzer analyzer, boolean isOutputTuple) { + TupleDescriptor result = + analyzer.getDescTbl().createTupleDescriptor( + tupleDebugName() + (isOutputTuple ? "-out" : "-intermed")); + List<Expr> exprs = Lists.newArrayListWithCapacity( + groupingExprs_.size() + aggregateExprs_.size()); + exprs.addAll(groupingExprs_); + exprs.addAll(aggregateExprs_); + + int aggregateExprStartIndex = groupingExprs_.size(); + for (int i = 0; i < exprs.size(); ++i) { + Expr expr = exprs.get(i); + SlotDescriptor slotDesc = analyzer.addSlotDescriptor(result); + slotDesc.initFromExpr(expr); + if (i < aggregateExprStartIndex) { + // register equivalence between grouping slot and grouping expr; + // do this only when the grouping expr isn't a constant, otherwise + // it'll simply show up as a gratuitous HAVING predicate + // (which would actually be incorrect if the constant happens to be NULL) + if (!expr.isConstant()) { + analyzer.createAuxEquivPredicate(new SlotRef(slotDesc), expr.clone()); + } + } else { + Preconditions.checkArgument(expr instanceof FunctionCallExpr); + FunctionCallExpr aggExpr = (FunctionCallExpr)expr; + if (aggExpr.isMergeAggFn()) { + slotDesc.setLabel(aggExpr.getChild(0).toSql()); + slotDesc.setSourceExpr(aggExpr.getChild(0)); + } else { + slotDesc.setLabel(aggExpr.toSql()); + slotDesc.setSourceExpr(aggExpr); + } + + // count(*) is non-nullable. + if (aggExpr.getFnName().getFunction().equals("count")) { + // TODO: Consider making nullability a property of types or of builtin agg fns. + // row_number, rank, and dense_rank are non-nullable as well. + slotDesc.setIsNullable(false); + } + if (!isOutputTuple) { + Type intermediateType = ((AggregateFunction)aggExpr.fn_).getIntermediateType(); + if (intermediateType != null) { + // Use the output type as intermediate if the function has a wildcard decimal. + if (!intermediateType.isWildcardDecimal()) { + slotDesc.setType(intermediateType); + } else { + Preconditions.checkState(expr.getType().isDecimal()); + } + } + } + } + } + String prefix = (isOutputTuple ? "result " : "intermediate "); + LOG.trace(prefix + " tuple=" + result.debugString()); + return result; + } + + /** + * Marks the slots required for evaluating an Analytic/AggregateInfo by + * resolving the materialized aggregate/analytic exprs against smap, + * and then marking their slots. + */ + public abstract void materializeRequiredSlots(Analyzer analyzer, + ExprSubstitutionMap smap); + + public ArrayList<Expr> getGroupingExprs() { return groupingExprs_; } + public ArrayList<FunctionCallExpr> getAggregateExprs() { return aggregateExprs_; } + public TupleDescriptor getOutputTupleDesc() { return outputTupleDesc_; } + public TupleDescriptor getIntermediateTupleDesc() { return intermediateTupleDesc_; } + public TupleId getIntermediateTupleId() { return intermediateTupleDesc_.getId(); } + public TupleId getOutputTupleId() { return outputTupleDesc_.getId(); } + public boolean requiresIntermediateTuple() { + Preconditions.checkNotNull(intermediateTupleDesc_); + Preconditions.checkNotNull(outputTupleDesc_); + return intermediateTupleDesc_ != outputTupleDesc_; + } + + /** + * Returns true if evaluating the given aggregate exprs requires an intermediate tuple, + * i.e., whether one of the aggregate functions has an intermediate type different from + * its output type. + */ + public static <T extends Expr> boolean requiresIntermediateTuple(List<T> aggExprs) { + for (Expr aggExpr: aggExprs) { + Type intermediateType = ((AggregateFunction) aggExpr.fn_).getIntermediateType(); + if (intermediateType != null) return true; + } + return false; + } + + public String debugString() { + StringBuilder out = new StringBuilder(); + out.append(Objects.toStringHelper(this) + .add("grouping_exprs", Expr.debugString(groupingExprs_)) + .add("aggregate_exprs", Expr.debugString(aggregateExprs_)) + .add("intermediate_tuple", (intermediateTupleDesc_ == null) + ? "null" : intermediateTupleDesc_.debugString()) + .add("output_tuple", (outputTupleDesc_ == null) + ? "null" : outputTupleDesc_.debugString()) + .toString()); + return out.toString(); + } + + protected abstract String tupleDebugName(); +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/AlterTableAddPartitionStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableAddPartitionStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableAddPartitionStmt.java new file mode 100644 index 0000000..a5cb2ca --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableAddPartitionStmt.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.authorization.Privilege; +import com.cloudera.impala.catalog.HdfsTable; +import com.cloudera.impala.catalog.Table; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.common.FileSystemUtil; +import com.cloudera.impala.thrift.TAlterTableAddPartitionParams; +import com.cloudera.impala.thrift.TAlterTableParams; +import com.cloudera.impala.thrift.TAlterTableType; +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.permission.FsAction; + +/** + * Represents an ALTER TABLE ADD PARTITION statement. + */ +public class AlterTableAddPartitionStmt extends AlterTableStmt { + private final HdfsUri location_; + private final boolean ifNotExists_; + private final PartitionSpec partitionSpec_; + private final HdfsCachingOp cacheOp_; + + public AlterTableAddPartitionStmt(TableName tableName, + PartitionSpec partitionSpec, HdfsUri location, boolean ifNotExists, + HdfsCachingOp cacheOp) { + super(tableName); + Preconditions.checkState(partitionSpec != null); + location_ = location; + ifNotExists_ = ifNotExists; + partitionSpec_ = partitionSpec; + partitionSpec_.setTableName(tableName); + cacheOp_ = cacheOp; + } + + public boolean getIfNotExists() { return ifNotExists_; } + public HdfsUri getLocation() { return location_; } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder("ALTER TABLE " + getTbl()); + sb.append(" ADD "); + if (ifNotExists_) { + sb.append("IF NOT EXISTS "); + } + sb.append(" " + partitionSpec_.toSql()); + if (location_ != null) { + sb.append(String.format(" LOCATION '%s'", location_)); + } + if (cacheOp_ != null) sb.append(cacheOp_.toSql()); + return sb.toString(); + } + + @Override + public TAlterTableParams toThrift() { + TAlterTableParams params = super.toThrift(); + params.setAlter_type(TAlterTableType.ADD_PARTITION); + TAlterTableAddPartitionParams addPartParams = new TAlterTableAddPartitionParams(); + addPartParams.setPartition_spec(partitionSpec_.toThrift()); + addPartParams.setLocation(location_ == null ? null : location_.toString()); + addPartParams.setIf_not_exists(ifNotExists_); + if (cacheOp_ != null) addPartParams.setCache_op(cacheOp_.toThrift()); + params.setAdd_partition_params(addPartParams); + return params; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + super.analyze(analyzer); + if (!ifNotExists_) partitionSpec_.setPartitionShouldNotExist(); + partitionSpec_.setPrivilegeRequirement(Privilege.ALTER); + partitionSpec_.analyze(analyzer); + + if (location_ != null) { + location_.analyze(analyzer, Privilege.ALL, FsAction.READ_WRITE); + } + + boolean shouldCache = false; + Table table = getTargetTable(); + if (cacheOp_ != null) { + cacheOp_.analyze(analyzer); + shouldCache = cacheOp_.shouldCache(); + } else if (table instanceof HdfsTable) { + shouldCache = ((HdfsTable)table).isMarkedCached(); + } + if (shouldCache) { + if (!(table instanceof HdfsTable)) { + throw new AnalysisException("Caching must target a HDFS table: " + + table.getFullName()); + } + HdfsTable hdfsTable = (HdfsTable)table; + if ((location_ != null && !FileSystemUtil.isPathCacheable(location_.getPath())) || + (location_ == null && !hdfsTable.isLocationCacheable())) { + throw new AnalysisException(String.format("Location '%s' cannot be cached. " + + "Please retry without caching: ALTER TABLE %s ADD PARTITION ... UNCACHED", + (location_ != null) ? location_.toString() : hdfsTable.getLocation(), + table.getFullName())); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/AlterTableAddReplaceColsStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableAddReplaceColsStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableAddReplaceColsStmt.java new file mode 100644 index 0000000..aaa223a --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableAddReplaceColsStmt.java @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +import com.cloudera.impala.catalog.Column; +import com.cloudera.impala.catalog.HBaseTable; +import com.cloudera.impala.catalog.Table; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TAlterTableAddReplaceColsParams; +import com.cloudera.impala.thrift.TAlterTableParams; +import com.cloudera.impala.thrift.TAlterTableType; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +/** + * Represents an ALTER TABLE ADD|REPLACE COLUMNS (colDef1, colDef2, ...) statement. + */ +public class AlterTableAddReplaceColsStmt extends AlterTableStmt { + private final List<ColumnDef> columnDefs_; + private final boolean replaceExistingCols_; + + public AlterTableAddReplaceColsStmt(TableName tableName, List<ColumnDef> columnDefs, + boolean replaceExistingCols) { + super(tableName); + Preconditions.checkState(columnDefs != null && columnDefs.size() > 0); + columnDefs_ = Lists.newArrayList(columnDefs); + replaceExistingCols_ = replaceExistingCols; + } + + public List<ColumnDef> getColumnDescs() { return columnDefs_; } + + // Replace columns instead of appending new columns. + public boolean getReplaceExistingCols() { + return replaceExistingCols_; + } + + @Override + public TAlterTableParams toThrift() { + TAlterTableParams params = super.toThrift(); + params.setAlter_type(TAlterTableType.ADD_REPLACE_COLUMNS); + TAlterTableAddReplaceColsParams colParams = new TAlterTableAddReplaceColsParams(); + for (ColumnDef col: getColumnDescs()) { + colParams.addToColumns(col.toThrift()); + } + colParams.setReplace_existing_cols(replaceExistingCols_); + params.setAdd_replace_cols_params(colParams); + return params; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + super.analyze(analyzer); + Table t = getTargetTable(); + // TODO: Support column-level DDL on HBase tables. Requires updating the column + // mappings along with the table columns. + if (t instanceof HBaseTable) { + throw new AnalysisException("ALTER TABLE ADD|REPLACE COLUMNS not currently " + + "supported on HBase tables."); + } + + // Build a set of the partition keys for the table. + Set<String> existingPartitionKeys = Sets.newHashSet(); + for (FieldSchema fs: t.getMetaStoreTable().getPartitionKeys()) { + existingPartitionKeys.add(fs.getName().toLowerCase()); + } + + // Make sure the new columns don't already exist in the table, that the names + // are all valid and unique, and that none of the columns conflict with + // partition columns. + Set<String> colNames = Sets.newHashSet(); + for (ColumnDef c: columnDefs_) { + c.analyze(); + String colName = c.getColName().toLowerCase(); + if (existingPartitionKeys.contains(colName)) { + throw new AnalysisException( + "Column name conflicts with existing partition column: " + colName); + } + + Column col = t.getColumn(colName); + if (col != null && !replaceExistingCols_) { + throw new AnalysisException("Column already exists: " + colName); + } else if (!colNames.add(colName)) { + throw new AnalysisException("Duplicate column name: " + colName); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/AlterTableChangeColStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableChangeColStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableChangeColStmt.java new file mode 100644 index 0000000..c733ca0 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableChangeColStmt.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +import com.cloudera.impala.catalog.HBaseTable; +import com.cloudera.impala.catalog.Table; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TAlterTableChangeColParams; +import com.cloudera.impala.thrift.TAlterTableParams; +import com.cloudera.impala.thrift.TAlterTableType; +import com.google.common.base.Preconditions; + +/** + * Represents an ALTER TABLE CHANGE COLUMN colName newColDef statement. + * Note: It would be fairly simple to reuse this class to support ALTER TABLE MODIFY + * newColDef statements in the future my making colName optional. + */ +public class AlterTableChangeColStmt extends AlterTableStmt { + private final String colName_; + private final ColumnDef newColDef_; + + public AlterTableChangeColStmt(TableName tableName, String colName, + ColumnDef newColDef) { + super(tableName); + Preconditions.checkNotNull(newColDef); + Preconditions.checkState(colName != null && !colName.isEmpty()); + colName_ = colName; + newColDef_ = newColDef; + } + + public String getColName() { return colName_; } + public ColumnDef getNewColDef() { return newColDef_; } + + @Override + public TAlterTableParams toThrift() { + TAlterTableParams params = super.toThrift(); + params.setAlter_type(TAlterTableType.CHANGE_COLUMN); + TAlterTableChangeColParams colParams = new TAlterTableChangeColParams(); + colParams.setCol_name(colName_); + colParams.setNew_col_def(newColDef_.toThrift()); + params.setChange_col_params(colParams); + return params; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + super.analyze(analyzer); + Table t = getTargetTable(); + // TODO: Support column-level DDL on HBase tables. Requires updating the column + // mappings along with the table columns. + if (t instanceof HBaseTable) { + throw new AnalysisException("ALTER TABLE CHANGE COLUMN not currently supported " + + "on HBase tables."); + } + String tableName = getDb() + "." + getTbl(); + + // Verify there are no conflicts with partition columns. + for (FieldSchema fs: t.getMetaStoreTable().getPartitionKeys()) { + if (fs.getName().toLowerCase().equals(colName_.toLowerCase())) { + throw new AnalysisException("Cannot modify partition column: " + colName_); + } + if (fs.getName().toLowerCase().equals(newColDef_.getColName().toLowerCase())) { + throw new AnalysisException( + "Column name conflicts with existing partition column: " + + newColDef_.getColName()); + } + } + + // Verify the column being modified exists in the table + if (t.getColumn(colName_) == null) { + throw new AnalysisException(String.format( + "Column '%s' does not exist in table: %s", colName_, tableName)); + } + + // Check that the new column def's name is valid. + newColDef_.analyze(); + // Verify that if the column name is being changed, the new name doesn't conflict + // with an existing column. + if (!colName_.toLowerCase().equals(newColDef_.getColName().toLowerCase()) && + t.getColumn(newColDef_.getColName()) != null) { + throw new AnalysisException("Column already exists: " + newColDef_.getColName()); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/AlterTableDropColStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableDropColStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableDropColStmt.java new file mode 100644 index 0000000..d7f5ab5 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableDropColStmt.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +import com.cloudera.impala.catalog.HBaseTable; +import com.cloudera.impala.catalog.Table; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TAlterTableDropColParams; +import com.cloudera.impala.thrift.TAlterTableParams; +import com.cloudera.impala.thrift.TAlterTableType; +import com.google.common.base.Preconditions; + +/** + * Represents an ALTER TABLE DROP COLUMN statement. + * Note: Hive does not support this syntax for droppping columns, but it is supported + * by mysql. + */ +public class AlterTableDropColStmt extends AlterTableStmt { + private final String colName_; + + public AlterTableDropColStmt(TableName tableName, String colName) { + super(tableName); + Preconditions.checkState(colName != null && !colName.isEmpty()); + colName_ = colName; + } + + public String getColName() { return colName_; } + + @Override + public TAlterTableParams toThrift() { + TAlterTableParams params = super.toThrift(); + params.setAlter_type(TAlterTableType.DROP_COLUMN); + TAlterTableDropColParams dropColParams = new TAlterTableDropColParams(colName_); + params.setDrop_col_params(dropColParams); + return params; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + super.analyze(analyzer); + Table t = getTargetTable(); + // TODO: Support column-level DDL on HBase tables. Requires updating the column + // mappings along with the table columns. + if (t instanceof HBaseTable) { + throw new AnalysisException("ALTER TABLE DROP COLUMN not currently supported " + + "on HBase tables."); + } + String tableName = getDb() + "." + getTbl(); + + for (FieldSchema fs: t.getMetaStoreTable().getPartitionKeys()) { + if (fs.getName().toLowerCase().equals(colName_.toLowerCase())) { + throw new AnalysisException("Cannot drop partition column: " + fs.getName()); + } + } + + if (t.getColumns().size() - t.getMetaStoreTable().getPartitionKeysSize() <= 1) { + throw new AnalysisException(String.format( + "Cannot drop column '%s' from %s. Tables must contain at least 1 column.", + colName_, tableName)); + } + + if (t.getColumn(colName_) == null) { + throw new AnalysisException(String.format( + "Column '%s' does not exist in table: %s", colName_, tableName)); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java new file mode 100644 index 0000000..f8bc09c --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.authorization.Privilege; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TAlterTableDropPartitionParams; +import com.cloudera.impala.thrift.TAlterTableParams; +import com.cloudera.impala.thrift.TAlterTableType; +import com.google.common.base.Preconditions; + +/** + * Represents an ALTER TABLE DROP PARTITION statement. + */ +public class AlterTableDropPartitionStmt extends AlterTableStmt { + private final boolean ifExists_; + private final PartitionSpec partitionSpec_; + + // Setting this value causes dropped partition(s) to be permanently + // deleted. For example, for HDFS tables it skips the trash mechanism + private final boolean purgePartition_; + + public AlterTableDropPartitionStmt(TableName tableName, + PartitionSpec partitionSpec, boolean ifExists, boolean purgePartition) { + super(tableName); + Preconditions.checkNotNull(partitionSpec); + partitionSpec_ = partitionSpec; + partitionSpec_.setTableName(tableName); + ifExists_ = ifExists; + purgePartition_ = purgePartition; + } + + public boolean getIfNotExists() { return ifExists_; } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder("ALTER TABLE " + getTbl()); + sb.append(" DROP "); + if (ifExists_) sb.append("IF EXISTS "); + sb.append(" DROP " + partitionSpec_.toSql()); + if (purgePartition_) sb.append(" PURGE"); + return sb.toString(); + } + + @Override + public TAlterTableParams toThrift() { + TAlterTableParams params = super.toThrift(); + params.setAlter_type(TAlterTableType.DROP_PARTITION); + TAlterTableDropPartitionParams addPartParams = new TAlterTableDropPartitionParams(); + addPartParams.setPartition_spec(partitionSpec_.toThrift()); + addPartParams.setIf_exists(ifExists_); + addPartParams.setPurge(purgePartition_); + params.setDrop_partition_params(addPartParams); + return params; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + super.analyze(analyzer); + if (!ifExists_) partitionSpec_.setPartitionShouldExist(); + partitionSpec_.setPrivilegeRequirement(Privilege.ALTER); + partitionSpec_.analyze(analyzer); + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/AlterTableOrViewRenameStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableOrViewRenameStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableOrViewRenameStmt.java new file mode 100644 index 0000000..009535c --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableOrViewRenameStmt.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.authorization.Privilege; +import com.cloudera.impala.catalog.View; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TAccessEvent; +import com.cloudera.impala.thrift.TAlterTableOrViewRenameParams; +import com.cloudera.impala.thrift.TAlterTableParams; +import com.cloudera.impala.thrift.TAlterTableType; +import com.cloudera.impala.thrift.TTableName; +import com.google.common.base.Preconditions; + +/** + * Represents an ALTER TABLE/VIEW RENAME statement. + */ +public class AlterTableOrViewRenameStmt extends AlterTableStmt { + protected final TableName newTableName_; + + // Set during analysis + protected String newDbName_; + + // True if we are renaming a table. False if we are renaming a view. + protected final boolean renameTable_; + + public AlterTableOrViewRenameStmt(TableName oldTableName, TableName newTableName, + boolean renameTable) { + super(oldTableName); + Preconditions.checkState(newTableName != null && !newTableName.isEmpty()); + newTableName_ = newTableName; + renameTable_ = renameTable; + } + + public String getNewTbl() { + return newTableName_.getTbl(); + } + + public String getNewDb() { + Preconditions.checkNotNull(newDbName_); + return newDbName_; + } + + @Override + public TAlterTableParams toThrift() { + TAlterTableParams params = super.toThrift(); + params.setAlter_type( + (renameTable_) ? TAlterTableType.RENAME_TABLE : TAlterTableType.RENAME_VIEW); + TAlterTableOrViewRenameParams renameParams = + new TAlterTableOrViewRenameParams(new TTableName(getNewDb(), getNewTbl())); + params.setRename_params(renameParams); + return params; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + newTableName_.analyze(); + table_ = analyzer.getTable(tableName_, Privilege.ALTER); + if (table_ instanceof View && renameTable_) { + throw new AnalysisException(String.format( + "ALTER TABLE not allowed on a view: %s", table_.getFullName())); + } + if (!(table_ instanceof View) && !renameTable_) { + throw new AnalysisException(String.format( + "ALTER VIEW not allowed on a table: %s", table_.getFullName())); + } + newDbName_ = analyzer.getTargetDbName(newTableName_); + if (analyzer.dbContainsTable(newDbName_, newTableName_.getTbl(), Privilege.CREATE)) { + throw new AnalysisException(Analyzer.TBL_ALREADY_EXISTS_ERROR_MSG + + String.format("%s.%s", newDbName_, getNewTbl())); + } + analyzer.addAccessEvent(new TAccessEvent(newDbName_ + "." + newTableName_.getTbl(), + table_.getCatalogObjectType(), Privilege.CREATE.toString())); + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/AlterTableRecoverPartitionsStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableRecoverPartitionsStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableRecoverPartitionsStmt.java new file mode 100644 index 0000000..c7e796c --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableRecoverPartitionsStmt.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.catalog.HdfsTable; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TAlterTableParams; +import com.cloudera.impala.thrift.TAlterTableType; + +/** + * Represents an ALTER TABLE RECOVER PARTITIONS statement. + */ +public class AlterTableRecoverPartitionsStmt extends AlterTableStmt { + + public AlterTableRecoverPartitionsStmt(TableName tableName) { + super(tableName); + } + + @Override + public TAlterTableParams toThrift() { + TAlterTableParams params = super.toThrift(); + params.setAlter_type(TAlterTableType.RECOVER_PARTITIONS); + return params; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + super.analyze(analyzer); + + // Make sure the target table is HdfsTable. + if (!(table_ instanceof HdfsTable)) { + throw new AnalysisException("ALTER TABLE RECOVER PARTITIONS " + + "must target an HDFS table: " + tableName_); + } + + // Make sure the target table is partitioned. + if (table_.getMetaStoreTable().getPartitionKeysSize() == 0) { + throw new AnalysisException("Table is not partitioned: " + tableName_); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/AlterTableSetCachedStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetCachedStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetCachedStmt.java new file mode 100644 index 0000000..f63cf3e --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetCachedStmt.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.catalog.HdfsTable; +import com.cloudera.impala.catalog.HdfsPartition; +import com.cloudera.impala.catalog.Table; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TAlterTableParams; +import com.cloudera.impala.thrift.TAlterTableSetCachedParams; +import com.cloudera.impala.thrift.TAlterTableType; +import com.google.common.base.Preconditions; + +/** + * Represents an ALTER TABLE [PARTITION partitionSpec] SET [UNCACHED|CACHED 'pool']. + */ +public class AlterTableSetCachedStmt extends AlterTableSetStmt { + private final HdfsCachingOp cacheOp_; + + public AlterTableSetCachedStmt(TableName tableName, + PartitionSpec partitionSpec, HdfsCachingOp cacheOp) { + super(tableName, partitionSpec); + Preconditions.checkNotNull(cacheOp); + cacheOp_ = cacheOp; + } + + @Override + public TAlterTableParams toThrift() { + TAlterTableParams params = super.toThrift(); + params.setAlter_type(TAlterTableType.SET_CACHED); + TAlterTableSetCachedParams cachingParams = + new TAlterTableSetCachedParams(); + if (getPartitionSpec() != null) { + cachingParams.setPartition_spec(getPartitionSpec().toThrift()); + } + cachingParams.setCache_op(cacheOp_.toThrift()); + params.setSet_cached_params(cachingParams); + return params; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + super.analyze(analyzer); + cacheOp_.analyze(analyzer); + + Table table = getTargetTable(); + Preconditions.checkNotNull(table); + if (!(table instanceof HdfsTable)) { + throw new AnalysisException("ALTER TABLE SET [CACHED|UNCACHED] must target an " + + "HDFS table: " + table.getFullName()); + } + + if (cacheOp_.shouldCache()) { + boolean isCacheable; + PartitionSpec partSpec = getPartitionSpec(); + HdfsTable hdfsTable = (HdfsTable)table; + StringBuilder nameSb = new StringBuilder(); + if (partSpec != null) { + HdfsPartition part = hdfsTable.getPartition(partSpec.getPartitionSpecKeyValues()); + if (part == null) { + throw new AnalysisException("Partition spec does not exist: " + + partSpec.toSql()); + } + isCacheable = part.isCacheable(); + nameSb.append("Partition (" + part.getPartitionName() + ")"); + } else { + isCacheable = hdfsTable.isCacheable(); + nameSb.append("Table ").append(table.getFullName()); + } + if (!isCacheable) { + throw new AnalysisException(nameSb.toString() + " cannot be cached. Please " + + "check if the table or partitions are on a filesystem which supports " + + "caching."); + } + } + } +}
