http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java ---------------------------------------------------------------------- diff --git a/ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java b/ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java new file mode 100644 index 0000000..a6a731b --- /dev/null +++ b/ext-data-source/test/src/main/java/org/apache/impala/extdatasource/AllTypesDataSource.java @@ -0,0 +1,323 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.extdatasource; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.util.List; +import java.util.UUID; + +import com.cloudera.impala.extdatasource.thrift.TBinaryPredicate; +import com.cloudera.impala.extdatasource.thrift.TCloseParams; +import com.cloudera.impala.extdatasource.thrift.TCloseResult; +import com.cloudera.impala.extdatasource.thrift.TColumnDesc; +import com.cloudera.impala.extdatasource.thrift.TGetNextParams; +import com.cloudera.impala.extdatasource.thrift.TGetNextResult; +import com.cloudera.impala.extdatasource.thrift.TOpenParams; +import com.cloudera.impala.extdatasource.thrift.TOpenResult; +import com.cloudera.impala.extdatasource.thrift.TPrepareParams; +import com.cloudera.impala.extdatasource.thrift.TPrepareResult; +import com.cloudera.impala.extdatasource.thrift.TRowBatch; +import com.cloudera.impala.extdatasource.thrift.TTableSchema; +import com.cloudera.impala.extdatasource.util.SerializationUtils; +import com.cloudera.impala.extdatasource.v1.ExternalDataSource; +import com.cloudera.impala.thrift.TColumnData; +import com.cloudera.impala.thrift.TColumnType; +import com.cloudera.impala.thrift.TPrimitiveType; +import com.cloudera.impala.thrift.TScalarType; +import com.cloudera.impala.thrift.TStatus; +import com.cloudera.impala.thrift.TErrorCode; +import com.cloudera.impala.thrift.TTypeNodeType; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + +/** + * Data source implementation for tests that can: + * (a) Accepts every other offered conjunct for testing planning (though predicates are + * not actually evaluated) and returns trivial data of all supported types for + * query tests. + * (b) Validate the predicates offered by Impala. + */ +public class AllTypesDataSource implements ExternalDataSource { + // Total number of rows to return + private static final int NUM_ROWS_RETURNED = 5000; + + // Change the size of the batches that are returned + private static final int INITIAL_BATCH_SIZE = 500; + private static final int BATCH_SIZE_INCREMENT = 100; + + private static final TStatus STATUS_OK = + new TStatus(TErrorCode.OK, Lists.<String>newArrayList()); + + private int currRow_; + private boolean eos_; + private int batchSize_; + private TTableSchema schema_; + private DataSourceState state_; + private String scanHandle_; + private String validatePredicatesResult_; + + // Enumerates the states of the data source. + private enum DataSourceState { + CREATED, + OPENED, + CLOSED + } + + public AllTypesDataSource() { + eos_ = false; + currRow_ = 0; + state_ = DataSourceState.CREATED; + } + + /** + * Accepts every other conjunct and returns the constant number of rows that + * is always returned. + */ + @Override + public TPrepareResult prepare(TPrepareParams params) { + Preconditions.checkState(state_ == DataSourceState.CREATED); + List<Integer> accepted = Lists.newArrayList(); + int numRowsReturned = 0; + if (validatePredicates(params.getPredicates())) { + // Indicate all predicates are applied because we return a dummy row with the + // result later to validate the result in tests. Impala shouldn't try to apply + // predicates to that dummy row. + for (int i = 0; i < params.getPredicatesSize(); ++i) accepted.add(i); + numRowsReturned = 1; + } else { + // Default behavior is to accept every other predicate. They are not actually + // applied, but we want to validate that Impala applies the correct predicates. + for (int i = 0; i < params.getPredicatesSize(); ++i) { + if (i % 2 == 0) accepted.add(i); + } + numRowsReturned = NUM_ROWS_RETURNED; + } + return new TPrepareResult(STATUS_OK) + .setAccepted_conjuncts(accepted) + .setNum_rows_estimate(numRowsReturned); + } + + /** + * If the predicate value (assuming STRING) starts with 'VALIDATE_PREDICATES##', + * we validate the TPrepareParams.predicates against predicates specified after the + * 'VALIDATE_PREDICATES##' and return true. The result of the validation is stored + * in validatePredicatesResult_. + * + * The expected predicates are specified in the form "{slot} {TComparisonOp} {val}", + * and conjunctive predicates are separated by '&&'. + * + * For example, the predicates_spec validates the predicates in the following query: + * select * from table_name + * where predicates_spec = 'x LT 1 && y GT 2' and + * x < 1 and + * 2 > y; + * + * Current limitations: + * - Disjunctive predicates are not supported (e.g. "expr1 or expr2") + * - Only INT is supported + */ + private boolean validatePredicates(List<List<TBinaryPredicate>> predicates) { + if (predicates == null || predicates.isEmpty()) return false; + TBinaryPredicate firstPredicate = predicates.get(0).get(0); + if (!firstPredicate.getValue().isSetString_val()) return false; + String colVal = firstPredicate.getValue().getString_val(); + if (!colVal.toUpperCase().startsWith("VALIDATE_PREDICATES##")) return false; + + String[] colValParts = colVal.split("##"); + Preconditions.checkArgument(colValParts.length == 2); + String[] expectedPredicates = colValParts[1].split("&&"); + Preconditions.checkArgument(expectedPredicates.length == predicates.size() - 1); + + String result = "SUCCESS"; + for (int i = 1; i < predicates.size(); ++i) { + String[] predicateParts = expectedPredicates[i - 1].trim().split(" "); + Preconditions.checkArgument(predicateParts.length == 3); + TBinaryPredicate predicate = + Iterables.getOnlyElement(predicates.get(i)); + Preconditions.checkArgument(predicate.getValue().isSetInt_val()); + + String slotName = predicate.getCol().getName().toUpperCase(); + int intVal = predicate.getValue().getInt_val(); + if (!predicateParts[0].toUpperCase().equals(slotName) || + !predicateParts[1].toUpperCase().equals(predicate.getOp().name()) || + !predicateParts[2].equals(Integer.toString(intVal))) { + result = "Failed predicate, expected=" + expectedPredicates[i - 1].trim() + + " actual=" + predicate.toString(); + } + } + validatePredicatesResult_ = result; + return true; + } + + /** + * Initializes the batch size and stores the table schema. + */ + @Override + public TOpenResult open(TOpenParams params) { + Preconditions.checkState(state_ == DataSourceState.CREATED); + state_ = DataSourceState.OPENED; + batchSize_ = INITIAL_BATCH_SIZE; + schema_ = params.getRow_schema(); + // Need to check validatePredicates again because the call in Prepare() was from + // the frontend and used a different instance of this data source class. + if (validatePredicates(params.getPredicates())) { + // If validating predicates, only one STRING column should be selected. + Preconditions.checkArgument(schema_.getColsSize() == 1); + TColumnDesc firstCol = schema_.getCols().get(0); + TColumnType firstType = firstCol.getType(); + Preconditions.checkState(firstType.getTypesSize() == 1); + Preconditions.checkState(firstType.types.get(0).getType() == TTypeNodeType.SCALAR); + Preconditions.checkArgument( + firstType.types.get(0).scalar_type.getType() == TPrimitiveType.STRING); + } + scanHandle_ = UUID.randomUUID().toString(); + return new TOpenResult(STATUS_OK).setScan_handle(scanHandle_); + } + + /** + * If validating predicates, returns a single row with the result of the validation. + * Otherwise returns row batches with generated rows based on the row index. Called + * multiple times, so the current row is stored between calls. Each row batch is a + * different size (not necessarily the size specified by TOpenParams.batch_size to + * ensure that Impala can handle unexpected batch sizes. + */ + @Override + public TGetNextResult getNext(TGetNextParams params) { + Preconditions.checkState(state_ == DataSourceState.OPENED); + Preconditions.checkArgument(params.getScan_handle().equals(scanHandle_)); + if (eos_) return new TGetNextResult(STATUS_OK).setEos(eos_); + + if (validatePredicatesResult_ != null) { + TColumnData colData = new TColumnData(); + colData.setIs_null(Lists.newArrayList(false)); + colData.setString_vals(Lists.newArrayList(validatePredicatesResult_)); + eos_ = true; + return new TGetNextResult(STATUS_OK).setEos(eos_) + .setRows(new TRowBatch().setCols(Lists.newArrayList(colData)).setNum_rows(1)); + } + + List<TColumnData> cols = Lists.newArrayList(); + for (int i = 0; i < schema_.getColsSize(); ++i) { + cols.add(new TColumnData().setIs_null(Lists.<Boolean>newArrayList())); + } + + int numAdded = 0; + while (currRow_ < NUM_ROWS_RETURNED && numAdded < batchSize_) { + addRow(cols); + ++numAdded; + ++currRow_; + } + + batchSize_ += BATCH_SIZE_INCREMENT; + if (currRow_ == NUM_ROWS_RETURNED) eos_ = true; + return new TGetNextResult(STATUS_OK).setEos(eos_) + .setRows(new TRowBatch().setCols(cols).setNum_rows(numAdded)); + } + + /** + * Adds a row to the set of columns. For all numeric types the value is set to the + * row index (mod the size for integer types). For strings it is just a string + * containing the row index and every 5th result is null. + */ + private void addRow(List<TColumnData> cols) { + for (int i = 0; i < cols.size(); ++i) { + TColumnDesc colDesc = schema_.getCols().get(i); + TColumnData colData = cols.get(i); + TColumnType type = colDesc.getType(); + if (type.types.get(0).getType() != TTypeNodeType.SCALAR) { + // Unsupported non-scalar type. + throw new UnsupportedOperationException("Unsupported column type: " + + type.types.get(0).getType()); + } + Preconditions.checkState(type.getTypesSize() == 1); + TScalarType scalarType = type.types.get(0).scalar_type; + switch (scalarType.type) { + case TINYINT: + colData.addToIs_null(false); + colData.addToByte_vals((byte) (currRow_ % 10)); + break; + case SMALLINT: + colData.addToIs_null(false); + colData.addToShort_vals((short) (currRow_ % 100)); + break; + case INT: + colData.addToIs_null(false); + colData.addToInt_vals(currRow_); + break; + case BIGINT: + colData.addToIs_null(false); + colData.addToLong_vals((long) currRow_ * 10); + break; + case DOUBLE: + colData.addToIs_null(false); + colData.addToDouble_vals(currRow_); + break; + case FLOAT: + colData.addToIs_null(false); + colData.addToDouble_vals((float) (1.1 * currRow_)); + break; + case STRING: + if (currRow_ % 5 == 0) { + colData.addToIs_null(true); + } else { + colData.addToIs_null(false); + colData.addToString_vals(String.valueOf(currRow_)); + } + break; + case BOOLEAN: + colData.addToIs_null(false); + colData.addToBool_vals(currRow_ % 2 == 0); + break; + case TIMESTAMP: + colData.addToIs_null(false); + colData.addToBinary_vals( + SerializationUtils.encodeTimestamp(new Timestamp(currRow_))); + break; + case DECIMAL: + colData.addToIs_null(false); + BigInteger maxUnscaled = BigInteger.TEN.pow(scalarType.getPrecision()); + BigInteger val = maxUnscaled.subtract(BigInteger.valueOf(currRow_ + 1)); + val = val.mod(maxUnscaled); + if (currRow_ % 2 == 0) val = val.negate(); + colData.addToBinary_vals(SerializationUtils.encodeDecimal(new BigDecimal(val))); + break; + case BINARY: + case CHAR: + case DATE: + case DATETIME: + case INVALID_TYPE: + case NULL_TYPE: + default: + // Unsupported. + throw new UnsupportedOperationException("Unsupported column type: " + + scalarType.getType()); + } + } + } + + @Override + public TCloseResult close(TCloseParams params) { + Preconditions.checkState(state_ == DataSourceState.OPENED); + Preconditions.checkArgument(params.getScan_handle().equals(scanHandle_)); + state_ = DataSourceState.CLOSED; + return new TCloseResult(STATUS_OK); + } +}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/AggregateInfo.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AggregateInfo.java b/fe/src/main/java/com/cloudera/impala/analysis/AggregateInfo.java deleted file mode 100644 index e8e9445..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/AggregateInfo.java +++ /dev/null @@ -1,742 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.InternalException; -import com.cloudera.impala.planner.DataPartition; -import com.cloudera.impala.thrift.TPartitionType; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -/** - * Encapsulates all the information needed to compute the aggregate functions of a single - * Select block, including a possible 2nd phase aggregation step for DISTINCT aggregate - * functions and merge aggregation steps needed for distributed execution. - * - * The latter requires a tree structure of AggregateInfo objects which express the - * original aggregate computations as well as the necessary merging aggregate - * computations. - * TODO: get rid of this by transforming - * SELECT COUNT(DISTINCT a, b, ..) GROUP BY x, y, ... - * into an equivalent query with a inline view: - * SELECT COUNT(*) FROM (SELECT DISTINCT a, b, ..., x, y, ...) GROUP BY x, y, ... - * - * The tree structure looks as follows: - * - for non-distinct aggregation: - * - aggInfo: contains the original aggregation functions and grouping exprs - * - aggInfo.mergeAggInfo: contains the merging aggregation functions (grouping - * exprs are identical) - * - for distinct aggregation (for an explanation of the phases, see - * SelectStmt.createDistinctAggInfo()): - * - aggInfo: contains the phase 1 aggregate functions and grouping exprs - * - aggInfo.2ndPhaseDistinctAggInfo: contains the phase 2 aggregate functions and - * grouping exprs - * - aggInfo.mergeAggInfo: contains the merging aggregate functions for the phase 1 - * computation (grouping exprs are identical) - * - aggInfo.2ndPhaseDistinctAggInfo.mergeAggInfo: contains the merging aggregate - * functions for the phase 2 computation (grouping exprs are identical) - * - * In general, merging aggregate computations are idempotent; in other words, - * aggInfo.mergeAggInfo == aggInfo.mergeAggInfo.mergeAggInfo. - * - * TODO: move the merge construction logic from SelectStmt into AggregateInfo - * TODO: Add query tests for aggregation with intermediate tuples with num_nodes=1. - */ -public class AggregateInfo extends AggregateInfoBase { - private final static Logger LOG = LoggerFactory.getLogger(AggregateInfo.class); - - public enum AggPhase { - FIRST, - FIRST_MERGE, - SECOND, - SECOND_MERGE; - - public boolean isMerge() { return this == FIRST_MERGE || this == SECOND_MERGE; } - }; - - // created by createMergeAggInfo() - private AggregateInfo mergeAggInfo_; - - // created by createDistinctAggInfo() - private AggregateInfo secondPhaseDistinctAggInfo_; - - private final AggPhase aggPhase_; - - // Map from all grouping and aggregate exprs to a SlotRef referencing the corresp. slot - // in the intermediate tuple. Identical to outputTupleSmap_ if no aggregateExpr has an - // output type that is different from its intermediate type. - protected ExprSubstitutionMap intermediateTupleSmap_ = new ExprSubstitutionMap(); - - // Map from all grouping and aggregate exprs to a SlotRef referencing the corresp. slot - // in the output tuple. - protected ExprSubstitutionMap outputTupleSmap_ = new ExprSubstitutionMap(); - - // Map from slots of outputTupleSmap_ to the corresponding slot in - // intermediateTupleSmap_. - protected ExprSubstitutionMap outputToIntermediateTupleSmap_ = - new ExprSubstitutionMap(); - - // if set, a subset of groupingExprs_; set and used during planning - private List<Expr> partitionExprs_; - - // C'tor creates copies of groupingExprs and aggExprs. - private AggregateInfo(ArrayList<Expr> groupingExprs, - ArrayList<FunctionCallExpr> aggExprs, AggPhase aggPhase) { - super(groupingExprs, aggExprs); - aggPhase_ = aggPhase; - } - - /** - * C'tor for cloning. - */ - private AggregateInfo(AggregateInfo other) { - super(other); - if (other.mergeAggInfo_ != null) { - mergeAggInfo_ = other.mergeAggInfo_.clone(); - } - if (other.secondPhaseDistinctAggInfo_ != null) { - secondPhaseDistinctAggInfo_ = other.secondPhaseDistinctAggInfo_.clone(); - } - aggPhase_ = other.aggPhase_; - outputTupleSmap_ = other.outputTupleSmap_.clone(); - if (other.requiresIntermediateTuple()) { - intermediateTupleSmap_ = other.intermediateTupleSmap_.clone(); - } else { - Preconditions.checkState(other.intermediateTupleDesc_ == other.outputTupleDesc_); - intermediateTupleSmap_ = outputTupleSmap_; - } - partitionExprs_ = - (other.partitionExprs_ != null) ? Expr.cloneList(other.partitionExprs_) : null; - } - - public List<Expr> getPartitionExprs() { return partitionExprs_; } - public void setPartitionExprs(List<Expr> exprs) { partitionExprs_ = exprs; } - - /** - * Creates complete AggregateInfo for groupingExprs and aggExprs, including - * aggTupleDesc and aggTupleSMap. If parameter tupleDesc != null, sets aggTupleDesc to - * that instead of creating a new descriptor (after verifying that the passed-in - * descriptor is correct for the given aggregation). - * Also creates mergeAggInfo and secondPhaseDistinctAggInfo, if needed. - * If an aggTupleDesc is created, also registers eq predicates between the - * grouping exprs and their respective slots with 'analyzer'. - */ - static public AggregateInfo create( - ArrayList<Expr> groupingExprs, ArrayList<FunctionCallExpr> aggExprs, - TupleDescriptor tupleDesc, Analyzer analyzer) - throws AnalysisException { - Preconditions.checkState( - (groupingExprs != null && !groupingExprs.isEmpty()) - || (aggExprs != null && !aggExprs.isEmpty())); - Expr.removeDuplicates(groupingExprs); - Expr.removeDuplicates(aggExprs); - AggregateInfo result = new AggregateInfo(groupingExprs, aggExprs, AggPhase.FIRST); - - // collect agg exprs with DISTINCT clause - ArrayList<FunctionCallExpr> distinctAggExprs = Lists.newArrayList(); - if (aggExprs != null) { - for (FunctionCallExpr aggExpr: aggExprs) { - if (aggExpr.isDistinct()) distinctAggExprs.add(aggExpr); - } - } - - if (distinctAggExprs.isEmpty()) { - if (tupleDesc == null) { - result.createTupleDescs(analyzer); - result.createSmaps(analyzer); - } else { - // A tupleDesc should only be given for UNION DISTINCT. - Preconditions.checkState(aggExprs == null); - result.outputTupleDesc_ = tupleDesc; - result.intermediateTupleDesc_ = tupleDesc; - } - result.createMergeAggInfo(analyzer); - } else { - // we don't allow you to pass in a descriptor for distinct aggregation - // (we need two descriptors) - Preconditions.checkState(tupleDesc == null); - result.createDistinctAggInfo(groupingExprs, distinctAggExprs, analyzer); - } - LOG.debug("agg info:\n" + result.debugString()); - return result; - } - - /** - * Create aggregate info for select block containing aggregate exprs with - * DISTINCT clause. - * This creates: - * - aggTupleDesc - * - a complete secondPhaseDistinctAggInfo - * - mergeAggInfo - * - * At the moment, we require that all distinct aggregate - * functions be applied to the same set of exprs (ie, we can't do something - * like SELECT COUNT(DISTINCT id), COUNT(DISTINCT address)). - * Aggregation happens in two successive phases: - * - the first phase aggregates by all grouping exprs plus all parameter exprs - * of DISTINCT aggregate functions - * - * Example: - * SELECT a, COUNT(DISTINCT b, c), MIN(d), COUNT(*) FROM T GROUP BY a - * - 1st phase grouping exprs: a, b, c - * - 1st phase agg exprs: MIN(d), COUNT(*) - * - 2nd phase grouping exprs: a - * - 2nd phase agg exprs: COUNT(*), MIN(<MIN(d) from 1st phase>), - * SUM(<COUNT(*) from 1st phase>) - * - * TODO: expand implementation to cover the general case; this will require - * a different execution strategy - */ - private void createDistinctAggInfo( - ArrayList<Expr> origGroupingExprs, - ArrayList<FunctionCallExpr> distinctAggExprs, Analyzer analyzer) - throws AnalysisException { - Preconditions.checkState(!distinctAggExprs.isEmpty()); - // make sure that all DISTINCT params are the same; - // ignore top-level implicit casts in the comparison, we might have inserted - // those during analysis - ArrayList<Expr> expr0Children = Lists.newArrayList(); - - if (distinctAggExprs.get(0).getFnName().getFunction().equalsIgnoreCase( - "group_concat")) { - // Ignore separator parameter, otherwise the same would have to be present for all - // other distinct aggregates as well. - // TODO: Deal with constant exprs more generally, instead of special-casing - // group_concat(). - expr0Children.add(distinctAggExprs.get(0).getChild(0).ignoreImplicitCast()); - } else { - for (Expr expr : distinctAggExprs.get(0).getChildren()) { - expr0Children.add(expr.ignoreImplicitCast()); - } - } - for (int i = 1; i < distinctAggExprs.size(); ++i) { - ArrayList<Expr> exprIChildren = Lists.newArrayList(); - if (distinctAggExprs.get(i).getFnName().getFunction().equalsIgnoreCase( - "group_concat")) { - exprIChildren.add(distinctAggExprs.get(i).getChild(0).ignoreImplicitCast()); - } else { - for (Expr expr : distinctAggExprs.get(i).getChildren()) { - exprIChildren.add(expr.ignoreImplicitCast()); - } - } - if (!Expr.equalLists(expr0Children, exprIChildren)) { - throw new AnalysisException( - "all DISTINCT aggregate functions need to have the same set of " - + "parameters as " + distinctAggExprs.get(0).toSql() - + "; deviating function: " + distinctAggExprs.get(i).toSql()); - } - } - - // add DISTINCT parameters to grouping exprs - groupingExprs_.addAll(expr0Children); - - // remove DISTINCT aggregate functions from aggExprs - aggregateExprs_.removeAll(distinctAggExprs); - - createTupleDescs(analyzer); - createSmaps(analyzer); - createMergeAggInfo(analyzer); - createSecondPhaseAggInfo(origGroupingExprs, distinctAggExprs, analyzer); - } - - public AggregateInfo getMergeAggInfo() { return mergeAggInfo_; } - public AggregateInfo getSecondPhaseDistinctAggInfo() { - return secondPhaseDistinctAggInfo_; - } - public AggPhase getAggPhase() { return aggPhase_; } - public boolean isMerge() { return aggPhase_.isMerge(); } - public boolean isDistinctAgg() { return secondPhaseDistinctAggInfo_ != null; } - public ExprSubstitutionMap getIntermediateSmap() { return intermediateTupleSmap_; } - public ExprSubstitutionMap getOutputSmap() { return outputTupleSmap_; } - public ExprSubstitutionMap getOutputToIntermediateSmap() { - return outputToIntermediateTupleSmap_; - } - - public boolean hasAggregateExprs() { - return !aggregateExprs_.isEmpty() || - (secondPhaseDistinctAggInfo_ != null && - !secondPhaseDistinctAggInfo_.getAggregateExprs().isEmpty()); - } - - /** - * Return the tuple id produced in the final aggregation step. - */ - public TupleId getResultTupleId() { - if (isDistinctAgg()) return secondPhaseDistinctAggInfo_.getOutputTupleId(); - return getOutputTupleId(); - } - - public ArrayList<FunctionCallExpr> getMaterializedAggregateExprs() { - ArrayList<FunctionCallExpr> result = Lists.newArrayList(); - for (Integer i: materializedSlots_) { - result.add(aggregateExprs_.get(i)); - } - return result; - } - - /** - * Append ids of all slots that are being referenced in the process - * of performing the aggregate computation described by this AggregateInfo. - */ - public void getRefdSlots(List<SlotId> ids) { - Preconditions.checkState(outputTupleDesc_ != null); - if (groupingExprs_ != null) { - Expr.getIds(groupingExprs_, null, ids); - } - Expr.getIds(aggregateExprs_, null, ids); - // The backend assumes that the entire aggTupleDesc is materialized - for (int i = 0; i < outputTupleDesc_.getSlots().size(); ++i) { - ids.add(outputTupleDesc_.getSlots().get(i).getId()); - } - } - - /** - * Substitute all the expressions (grouping expr, aggregate expr) and update our - * substitution map according to the given substitution map: - * - smap typically maps from tuple t1 to tuple t2 (example: the smap of an - * inline view maps the virtual table ref t1 into a base table ref t2) - * - our grouping and aggregate exprs need to be substituted with the given - * smap so that they also reference t2 - * - aggTupleSMap needs to be recomputed to map exprs based on t2 - * onto our aggTupleDesc (ie, the left-hand side needs to be substituted with - * smap) - * - mergeAggInfo: this is not affected, because - * * its grouping and aggregate exprs only reference aggTupleDesc_ - * * its smap is identical to aggTupleSMap_ - * - 2ndPhaseDistinctAggInfo: - * * its grouping and aggregate exprs also only reference aggTupleDesc_ - * and are therefore not affected - * * its smap needs to be recomputed to map exprs based on t2 to its own - * aggTupleDesc - */ - public void substitute(ExprSubstitutionMap smap, Analyzer analyzer) - throws InternalException { - groupingExprs_ = Expr.substituteList(groupingExprs_, smap, analyzer, false); - LOG.trace("AggInfo: grouping_exprs=" + Expr.debugString(groupingExprs_)); - - // The smap in this case should not substitute the aggs themselves, only - // their subexpressions. - List<Expr> substitutedAggs = - Expr.substituteList(aggregateExprs_, smap, analyzer, false); - aggregateExprs_.clear(); - for (Expr substitutedAgg: substitutedAggs) { - aggregateExprs_.add((FunctionCallExpr) substitutedAgg); - } - - LOG.trace("AggInfo: agg_exprs=" + Expr.debugString(aggregateExprs_)); - outputTupleSmap_.substituteLhs(smap, analyzer); - intermediateTupleSmap_.substituteLhs(smap, analyzer); - if (secondPhaseDistinctAggInfo_ != null) { - secondPhaseDistinctAggInfo_.substitute(smap, analyzer); - } - } - - /** - * Create the info for an aggregation node that merges its pre-aggregated inputs: - * - pre-aggregation is computed by 'this' - * - tuple desc and smap are the same as that of the input (we're materializing - * the same logical tuple) - * - grouping exprs: slotrefs to the input's grouping slots - * - aggregate exprs: aggregation of the input's aggregateExprs slots - * - * The returned AggregateInfo shares its descriptor and smap with the input info; - * createAggTupleDesc() must not be called on it. - */ - private void createMergeAggInfo(Analyzer analyzer) { - Preconditions.checkState(mergeAggInfo_ == null); - TupleDescriptor inputDesc = intermediateTupleDesc_; - // construct grouping exprs - ArrayList<Expr> groupingExprs = Lists.newArrayList(); - for (int i = 0; i < getGroupingExprs().size(); ++i) { - SlotRef slotRef = new SlotRef(inputDesc.getSlots().get(i)); - groupingExprs.add(slotRef); - } - - // construct agg exprs - ArrayList<FunctionCallExpr> aggExprs = Lists.newArrayList(); - for (int i = 0; i < getAggregateExprs().size(); ++i) { - FunctionCallExpr inputExpr = getAggregateExprs().get(i); - Preconditions.checkState(inputExpr.isAggregateFunction()); - Expr aggExprParam = - new SlotRef(inputDesc.getSlots().get(i + getGroupingExprs().size())); - FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall( - inputExpr, Lists.newArrayList(aggExprParam)); - aggExpr.analyzeNoThrow(analyzer); - aggExprs.add(aggExpr); - } - - AggPhase aggPhase = - (aggPhase_ == AggPhase.FIRST) ? AggPhase.FIRST_MERGE : AggPhase.SECOND_MERGE; - mergeAggInfo_ = new AggregateInfo(groupingExprs, aggExprs, aggPhase); - mergeAggInfo_.intermediateTupleDesc_ = intermediateTupleDesc_; - mergeAggInfo_.outputTupleDesc_ = outputTupleDesc_; - mergeAggInfo_.intermediateTupleSmap_ = intermediateTupleSmap_; - mergeAggInfo_.outputTupleSmap_ = outputTupleSmap_; - mergeAggInfo_.materializedSlots_ = materializedSlots_; - } - - /** - * Creates an IF function call that returns NULL if any of the slots - * at indexes [firstIdx, lastIdx] return NULL. - * For example, the resulting IF function would like this for 3 slots: - * IF(IsNull(slot1), NULL, IF(IsNull(slot2), NULL, slot3)) - * Returns null if firstIdx is greater than lastIdx. - * Returns a SlotRef to the last slot if there is only one slot in range. - */ - private Expr createCountDistinctAggExprParam(int firstIdx, int lastIdx, - ArrayList<SlotDescriptor> slots) { - if (firstIdx > lastIdx) return null; - - Expr elseExpr = new SlotRef(slots.get(lastIdx)); - if (firstIdx == lastIdx) return elseExpr; - - for (int i = lastIdx - 1; i >= firstIdx; --i) { - ArrayList<Expr> ifArgs = Lists.newArrayList(); - SlotRef slotRef = new SlotRef(slots.get(i)); - // Build expr: IF(IsNull(slotRef), NULL, elseExpr) - Expr isNullPred = new IsNullPredicate(slotRef, false); - ifArgs.add(isNullPred); - ifArgs.add(new NullLiteral()); - ifArgs.add(elseExpr); - elseExpr = new FunctionCallExpr("if", ifArgs); - } - return elseExpr; - } - - /** - * Create the info for an aggregation node that computes the second phase of - * DISTINCT aggregate functions. - * (Refer to createDistinctAggInfo() for an explanation of the phases.) - * - 'this' is the phase 1 aggregation - * - grouping exprs are those of the original query (param origGroupingExprs) - * - aggregate exprs for the DISTINCT agg fns: these are aggregating the grouping - * slots that were added to the original grouping slots in phase 1; - * count is mapped to count(*) and sum is mapped to sum - * - other aggregate exprs: same as the non-DISTINCT merge case - * (count is mapped to sum, everything else stays the same) - * - * This call also creates the tuple descriptor and smap for the returned AggregateInfo. - */ - private void createSecondPhaseAggInfo( - ArrayList<Expr> origGroupingExprs, - ArrayList<FunctionCallExpr> distinctAggExprs, Analyzer analyzer) - throws AnalysisException { - Preconditions.checkState(secondPhaseDistinctAggInfo_ == null); - Preconditions.checkState(!distinctAggExprs.isEmpty()); - // The output of the 1st phase agg is the 1st phase intermediate. - TupleDescriptor inputDesc = intermediateTupleDesc_; - - // construct agg exprs for original DISTINCT aggregate functions - // (these aren't part of aggExprs_) - ArrayList<FunctionCallExpr> secondPhaseAggExprs = Lists.newArrayList(); - for (FunctionCallExpr inputExpr: distinctAggExprs) { - Preconditions.checkState(inputExpr.isAggregateFunction()); - FunctionCallExpr aggExpr = null; - if (inputExpr.getFnName().getFunction().equals("count")) { - // COUNT(DISTINCT ...) -> - // COUNT(IF(IsNull(<agg slot 1>), NULL, IF(IsNull(<agg slot 2>), NULL, ...))) - // We need the nested IF to make sure that we do not count - // column-value combinations if any of the distinct columns are NULL. - // This behavior is consistent with MySQL. - Expr ifExpr = createCountDistinctAggExprParam(origGroupingExprs.size(), - origGroupingExprs.size() + inputExpr.getChildren().size() - 1, - inputDesc.getSlots()); - Preconditions.checkNotNull(ifExpr); - ifExpr.analyzeNoThrow(analyzer); - aggExpr = new FunctionCallExpr("count", Lists.newArrayList(ifExpr)); - } else if (inputExpr.getFnName().getFunction().equals("group_concat")) { - // Syntax: GROUP_CONCAT([DISTINCT] expression [, separator]) - ArrayList<Expr> exprList = Lists.newArrayList(); - // Add "expression" parameter. Need to get it from the inputDesc's slots so the - // tuple reference is correct. - exprList.add(new SlotRef(inputDesc.getSlots().get(origGroupingExprs.size()))); - // Check if user provided a custom separator - if (inputExpr.getChildren().size() == 2) exprList.add(inputExpr.getChild(1)); - aggExpr = new FunctionCallExpr(inputExpr.getFnName(), exprList); - } else { - // SUM(DISTINCT <expr>) -> SUM(<last grouping slot>); - // (MIN(DISTINCT ...) and MAX(DISTINCT ...) have their DISTINCT turned - // off during analysis, and AVG() is changed to SUM()/COUNT()) - Expr aggExprParam = - new SlotRef(inputDesc.getSlots().get(origGroupingExprs.size())); - aggExpr = new FunctionCallExpr(inputExpr.getFnName(), - Lists.newArrayList(aggExprParam)); - } - secondPhaseAggExprs.add(aggExpr); - } - - // map all the remaining agg fns - for (int i = 0; i < aggregateExprs_.size(); ++i) { - FunctionCallExpr inputExpr = aggregateExprs_.get(i); - Preconditions.checkState(inputExpr.isAggregateFunction()); - // we're aggregating an intermediate slot of the 1st agg phase - Expr aggExprParam = - new SlotRef(inputDesc.getSlots().get(i + getGroupingExprs().size())); - FunctionCallExpr aggExpr = FunctionCallExpr.createMergeAggCall( - inputExpr, Lists.newArrayList(aggExprParam)); - secondPhaseAggExprs.add(aggExpr); - } - Preconditions.checkState( - secondPhaseAggExprs.size() == aggregateExprs_.size() + distinctAggExprs.size()); - - for (FunctionCallExpr aggExpr: secondPhaseAggExprs) { - aggExpr.analyzeNoThrow(analyzer); - Preconditions.checkState(aggExpr.isAggregateFunction()); - } - - ArrayList<Expr> substGroupingExprs = - Expr.substituteList(origGroupingExprs, intermediateTupleSmap_, analyzer, false); - secondPhaseDistinctAggInfo_ = - new AggregateInfo(substGroupingExprs, secondPhaseAggExprs, AggPhase.SECOND); - secondPhaseDistinctAggInfo_.createTupleDescs(analyzer); - secondPhaseDistinctAggInfo_.createSecondPhaseAggSMap(this, distinctAggExprs); - secondPhaseDistinctAggInfo_.createMergeAggInfo(analyzer); - } - - /** - * Create smap to map original grouping and aggregate exprs onto output - * of secondPhaseDistinctAggInfo. - */ - private void createSecondPhaseAggSMap( - AggregateInfo inputAggInfo, ArrayList<FunctionCallExpr> distinctAggExprs) { - outputTupleSmap_.clear(); - int slotIdx = 0; - ArrayList<SlotDescriptor> slotDescs = outputTupleDesc_.getSlots(); - - int numDistinctParams = distinctAggExprs.get(0).getChildren().size(); - // If we are counting distinct params of group_concat, we cannot include the custom - // separator since it is not a distinct param. - if (distinctAggExprs.get(0).getFnName().getFunction().equalsIgnoreCase( - "group_concat") - && numDistinctParams == 2) { - --numDistinctParams; - } - int numOrigGroupingExprs = - inputAggInfo.getGroupingExprs().size() - numDistinctParams; - Preconditions.checkState(slotDescs.size() == - numOrigGroupingExprs + distinctAggExprs.size() + - inputAggInfo.getAggregateExprs().size()); - - // original grouping exprs -> first m slots - for (int i = 0; i < numOrigGroupingExprs; ++i, ++slotIdx) { - Expr groupingExpr = inputAggInfo.getGroupingExprs().get(i); - outputTupleSmap_.put( - groupingExpr.clone(), new SlotRef(slotDescs.get(slotIdx))); - } - - // distinct agg exprs -> next n slots - for (int i = 0; i < distinctAggExprs.size(); ++i, ++slotIdx) { - Expr aggExpr = distinctAggExprs.get(i); - outputTupleSmap_.put( - aggExpr.clone(), (new SlotRef(slotDescs.get(slotIdx)))); - } - - // remaining agg exprs -> remaining slots - for (int i = 0; i < inputAggInfo.getAggregateExprs().size(); ++i, ++slotIdx) { - Expr aggExpr = inputAggInfo.getAggregateExprs().get(i); - outputTupleSmap_.put(aggExpr.clone(), new SlotRef(slotDescs.get(slotIdx))); - } - } - - /** - * Populates the output and intermediate smaps based on the output and intermediate - * tuples that are assumed to be set. If an intermediate tuple is required, also - * populates the output-to-intermediate smap and registers auxiliary equivalence - * predicates between the grouping slots of the two tuples. - */ - public void createSmaps(Analyzer analyzer) { - Preconditions.checkNotNull(outputTupleDesc_); - Preconditions.checkNotNull(intermediateTupleDesc_); - - List<Expr> exprs = Lists.newArrayListWithCapacity( - groupingExprs_.size() + aggregateExprs_.size()); - exprs.addAll(groupingExprs_); - exprs.addAll(aggregateExprs_); - for (int i = 0; i < exprs.size(); ++i) { - outputTupleSmap_.put(exprs.get(i).clone(), - new SlotRef(outputTupleDesc_.getSlots().get(i))); - if (!requiresIntermediateTuple()) continue; - intermediateTupleSmap_.put(exprs.get(i).clone(), - new SlotRef(intermediateTupleDesc_.getSlots().get(i))); - outputToIntermediateTupleSmap_.put( - new SlotRef(outputTupleDesc_.getSlots().get(i)), - new SlotRef(intermediateTupleDesc_.getSlots().get(i))); - if (i < groupingExprs_.size()) { - analyzer.createAuxEquivPredicate( - new SlotRef(outputTupleDesc_.getSlots().get(i)), - new SlotRef(intermediateTupleDesc_.getSlots().get(i))); - } - } - if (!requiresIntermediateTuple()) intermediateTupleSmap_ = outputTupleSmap_; - - LOG.trace("output smap=" + outputTupleSmap_.debugString()); - LOG.trace("intermediate smap=" + intermediateTupleSmap_.debugString()); - } - - /** - * Mark slots required for this aggregation as materialized: - * - all grouping output slots as well as grouping exprs - * - for non-distinct aggregation: the aggregate exprs of materialized aggregate slots; - * this assumes that the output slots corresponding to aggregate exprs have already - * been marked by the consumer of this select block - * - for distinct aggregation, we mark all aggregate output slots in order to keep - * things simple - * Also computes materializedAggregateExprs. - * This call must be idempotent because it may be called more than once for Union stmt. - */ - @Override - public void materializeRequiredSlots(Analyzer analyzer, ExprSubstitutionMap smap) { - for (int i = 0; i < groupingExprs_.size(); ++i) { - outputTupleDesc_.getSlots().get(i).setIsMaterialized(true); - intermediateTupleDesc_.getSlots().get(i).setIsMaterialized(true); - } - - // collect input exprs: grouping exprs plus aggregate exprs that need to be - // materialized - materializedSlots_.clear(); - List<Expr> exprs = Lists.newArrayList(); - exprs.addAll(groupingExprs_); - for (int i = 0; i < aggregateExprs_.size(); ++i) { - SlotDescriptor slotDesc = - outputTupleDesc_.getSlots().get(groupingExprs_.size() + i); - SlotDescriptor intermediateSlotDesc = - intermediateTupleDesc_.getSlots().get(groupingExprs_.size() + i); - if (isDistinctAgg()) { - slotDesc.setIsMaterialized(true); - intermediateSlotDesc.setIsMaterialized(true); - } - if (!slotDesc.isMaterialized()) continue; - intermediateSlotDesc.setIsMaterialized(true); - exprs.add(aggregateExprs_.get(i)); - materializedSlots_.add(i); - } - List<Expr> resolvedExprs = Expr.substituteList(exprs, smap, analyzer, false); - analyzer.materializeSlots(resolvedExprs); - - if (isDistinctAgg()) { - secondPhaseDistinctAggInfo_.materializeRequiredSlots(analyzer, null); - } - } - - /** - * Checks if all materialized aggregate expressions have distinct semantics. - * It returns true if either of the following is true: - * (1) all materialized aggregate expressions have distinct semantics - * (e.g. MIN, MAX, NDV). In other words, this optimization will work - * for COUNT(DISTINCT c) but not COUNT(c). - * (2) there are no aggregate expressions but only grouping expressions. - */ - public boolean hasAllDistinctAgg() { - if (hasAggregateExprs()) { - for (FunctionCallExpr aggExpr : getMaterializedAggregateExprs()) { - if (!aggExpr.isDistinct() && !aggExpr.ignoresDistinct()) return false; - } - } else { - Preconditions.checkState(!groupingExprs_.isEmpty()); - } - return true; - } - - /** - * Validates the internal state of this agg info: Checks that the number of - * materialized slots of the output tuple corresponds to the number of materialized - * aggregate functions plus the number of grouping exprs. Also checks that the return - * types of the aggregate and grouping exprs correspond to the slots in the output - * tuple. - */ - public void checkConsistency() { - ArrayList<SlotDescriptor> slots = outputTupleDesc_.getSlots(); - - // Check materialized slots. - int numMaterializedSlots = 0; - for (SlotDescriptor slotDesc: slots) { - if (slotDesc.isMaterialized()) ++numMaterializedSlots; - } - Preconditions.checkState(numMaterializedSlots == - materializedSlots_.size() + groupingExprs_.size()); - - // Check that grouping expr return types match the slot descriptors. - int slotIdx = 0; - for (int i = 0; i < groupingExprs_.size(); ++i) { - Expr groupingExpr = groupingExprs_.get(i); - Type slotType = slots.get(slotIdx).getType(); - Preconditions.checkState(groupingExpr.getType().equals(slotType), - String.format("Grouping expr %s returns type %s but its output tuple " + - "slot has type %s", groupingExpr.toSql(), - groupingExpr.getType().toString(), slotType.toString())); - ++slotIdx; - } - // Check that aggregate expr return types match the slot descriptors. - for (int i = 0; i < aggregateExprs_.size(); ++i) { - Expr aggExpr = aggregateExprs_.get(i); - Type slotType = slots.get(slotIdx).getType(); - Preconditions.checkState(aggExpr.getType().equals(slotType), - String.format("Agg expr %s returns type %s but its output tuple " + - "slot has type %s", aggExpr.toSql(), aggExpr.getType().toString(), - slotType.toString())); - ++slotIdx; - } - } - - /** - * Returns DataPartition derived from grouping exprs. - * Returns unpartitioned spec if no grouping. - * TODO: this won't work when we start supporting range partitions, - * because we could derive both hash and order-based partitions - */ - public DataPartition getPartition() { - if (groupingExprs_.isEmpty()) { - return DataPartition.UNPARTITIONED; - } else { - return DataPartition.hashPartitioned(groupingExprs_); - } - } - - @Override - public String debugString() { - StringBuilder out = new StringBuilder(super.debugString()); - out.append(Objects.toStringHelper(this) - .add("phase", aggPhase_) - .add("intermediate_smap", intermediateTupleSmap_.debugString()) - .add("output_smap", outputTupleSmap_.debugString()) - .toString()); - if (mergeAggInfo_ != this && mergeAggInfo_ != null) { - out.append("\nmergeAggInfo:\n" + mergeAggInfo_.debugString()); - } - if (secondPhaseDistinctAggInfo_ != null) { - out.append("\nsecondPhaseDistinctAggInfo:\n" - + secondPhaseDistinctAggInfo_.debugString()); - } - return out.toString(); - } - - @Override - protected String tupleDebugName() { return "agg-tuple"; } - - @Override - public AggregateInfo clone() { return new AggregateInfo(this); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/AggregateInfoBase.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AggregateInfoBase.java b/fe/src/main/java/com/cloudera/impala/analysis/AggregateInfoBase.java deleted file mode 100644 index f3ad3f8..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/AggregateInfoBase.java +++ /dev/null @@ -1,221 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.cloudera.impala.catalog.AggregateFunction; -import com.cloudera.impala.catalog.ColumnStats; -import com.cloudera.impala.catalog.Type; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -/** - * Base class for AggregateInfo and AnalyticInfo containing the intermediate and output - * tuple descriptors as well as their smaps for evaluating aggregate functions. - */ -public abstract class AggregateInfoBase { - private final static Logger LOG = - LoggerFactory.getLogger(AggregateInfoBase.class); - - // For aggregations: All unique grouping expressions from a select block. - // For analytics: Empty. - protected ArrayList<Expr> groupingExprs_; - - // For aggregations: All unique aggregate expressions from a select block. - // For analytics: The results of AnalyticExpr.getFnCall() for the unique - // AnalyticExprs of a select block. - protected ArrayList<FunctionCallExpr> aggregateExprs_; - - // The tuple into which the intermediate output of an aggregation is materialized. - // Contains groupingExprs.size() + aggregateExprs.size() slots, the first of which - // contain the values of the grouping exprs, followed by slots into which the - // aggregateExprs' update()/merge() symbols materialize their output, i.e., slots - // of the aggregate functions' intermediate types. - // Identical to outputTupleDesc_ if no aggregateExpr has an output type that is - // different from its intermediate type. - protected TupleDescriptor intermediateTupleDesc_; - - // The tuple into which the final output of the aggregation is materialized. - // Contains groupingExprs.size() + aggregateExprs.size() slots, the first of which - // contain the values of the grouping exprs, followed by slots into which the - // aggregateExprs' finalize() symbol write its result, i.e., slots of the aggregate - // functions' output types. - protected TupleDescriptor outputTupleDesc_; - - // For aggregation: indices into aggregate exprs for that need to be materialized - // For analytics: indices into the analytic exprs and their corresponding aggregate - // exprs that need to be materialized. - // Populated in materializeRequiredSlots() which must be implemented by subclasses. - protected ArrayList<Integer> materializedSlots_ = Lists.newArrayList(); - - protected AggregateInfoBase(ArrayList<Expr> groupingExprs, - ArrayList<FunctionCallExpr> aggExprs) { - Preconditions.checkState(groupingExprs != null || aggExprs != null); - groupingExprs_ = - groupingExprs != null ? Expr.cloneList(groupingExprs) : new ArrayList<Expr>(); - Preconditions.checkState(aggExprs != null || !(this instanceof AnalyticInfo)); - aggregateExprs_ = - aggExprs != null ? Expr.cloneList(aggExprs) : new ArrayList<FunctionCallExpr>(); - } - - /** - * C'tor for cloning. - */ - protected AggregateInfoBase(AggregateInfoBase other) { - groupingExprs_ = - (other.groupingExprs_ != null) ? Expr.cloneList(other.groupingExprs_) : null; - aggregateExprs_ = - (other.aggregateExprs_ != null) ? Expr.cloneList(other.aggregateExprs_) : null; - intermediateTupleDesc_ = other.intermediateTupleDesc_; - outputTupleDesc_ = other.outputTupleDesc_; - materializedSlots_ = Lists.newArrayList(other.materializedSlots_); - } - - /** - * Creates the intermediate and output tuple descriptors. If no agg expr has an - * intermediate type different from its output type, then only the output tuple - * descriptor is created and the intermediate tuple is set to the output tuple. - */ - protected void createTupleDescs(Analyzer analyzer) { - // Create the intermediate tuple desc first, so that the tuple ids are increasing - // from bottom to top in the plan tree. - intermediateTupleDesc_ = createTupleDesc(analyzer, false); - if (requiresIntermediateTuple(aggregateExprs_)) { - outputTupleDesc_ = createTupleDesc(analyzer, true); - } else { - outputTupleDesc_ = intermediateTupleDesc_; - } - } - - /** - * Returns a tuple descriptor for the aggregation/analytic's intermediate or final - * result, depending on whether isOutputTuple is true or false. - * Also updates the appropriate substitution map, and creates and registers auxiliary - * equality predicates between the grouping slots and the grouping exprs. - */ - private TupleDescriptor createTupleDesc(Analyzer analyzer, boolean isOutputTuple) { - TupleDescriptor result = - analyzer.getDescTbl().createTupleDescriptor( - tupleDebugName() + (isOutputTuple ? "-out" : "-intermed")); - List<Expr> exprs = Lists.newArrayListWithCapacity( - groupingExprs_.size() + aggregateExprs_.size()); - exprs.addAll(groupingExprs_); - exprs.addAll(aggregateExprs_); - - int aggregateExprStartIndex = groupingExprs_.size(); - for (int i = 0; i < exprs.size(); ++i) { - Expr expr = exprs.get(i); - SlotDescriptor slotDesc = analyzer.addSlotDescriptor(result); - slotDesc.initFromExpr(expr); - if (i < aggregateExprStartIndex) { - // register equivalence between grouping slot and grouping expr; - // do this only when the grouping expr isn't a constant, otherwise - // it'll simply show up as a gratuitous HAVING predicate - // (which would actually be incorrect if the constant happens to be NULL) - if (!expr.isConstant()) { - analyzer.createAuxEquivPredicate(new SlotRef(slotDesc), expr.clone()); - } - } else { - Preconditions.checkArgument(expr instanceof FunctionCallExpr); - FunctionCallExpr aggExpr = (FunctionCallExpr)expr; - if (aggExpr.isMergeAggFn()) { - slotDesc.setLabel(aggExpr.getChild(0).toSql()); - slotDesc.setSourceExpr(aggExpr.getChild(0)); - } else { - slotDesc.setLabel(aggExpr.toSql()); - slotDesc.setSourceExpr(aggExpr); - } - - // count(*) is non-nullable. - if (aggExpr.getFnName().getFunction().equals("count")) { - // TODO: Consider making nullability a property of types or of builtin agg fns. - // row_number, rank, and dense_rank are non-nullable as well. - slotDesc.setIsNullable(false); - } - if (!isOutputTuple) { - Type intermediateType = ((AggregateFunction)aggExpr.fn_).getIntermediateType(); - if (intermediateType != null) { - // Use the output type as intermediate if the function has a wildcard decimal. - if (!intermediateType.isWildcardDecimal()) { - slotDesc.setType(intermediateType); - } else { - Preconditions.checkState(expr.getType().isDecimal()); - } - } - } - } - } - String prefix = (isOutputTuple ? "result " : "intermediate "); - LOG.trace(prefix + " tuple=" + result.debugString()); - return result; - } - - /** - * Marks the slots required for evaluating an Analytic/AggregateInfo by - * resolving the materialized aggregate/analytic exprs against smap, - * and then marking their slots. - */ - public abstract void materializeRequiredSlots(Analyzer analyzer, - ExprSubstitutionMap smap); - - public ArrayList<Expr> getGroupingExprs() { return groupingExprs_; } - public ArrayList<FunctionCallExpr> getAggregateExprs() { return aggregateExprs_; } - public TupleDescriptor getOutputTupleDesc() { return outputTupleDesc_; } - public TupleDescriptor getIntermediateTupleDesc() { return intermediateTupleDesc_; } - public TupleId getIntermediateTupleId() { return intermediateTupleDesc_.getId(); } - public TupleId getOutputTupleId() { return outputTupleDesc_.getId(); } - public boolean requiresIntermediateTuple() { - Preconditions.checkNotNull(intermediateTupleDesc_); - Preconditions.checkNotNull(outputTupleDesc_); - return intermediateTupleDesc_ != outputTupleDesc_; - } - - /** - * Returns true if evaluating the given aggregate exprs requires an intermediate tuple, - * i.e., whether one of the aggregate functions has an intermediate type different from - * its output type. - */ - public static <T extends Expr> boolean requiresIntermediateTuple(List<T> aggExprs) { - for (Expr aggExpr: aggExprs) { - Type intermediateType = ((AggregateFunction) aggExpr.fn_).getIntermediateType(); - if (intermediateType != null) return true; - } - return false; - } - - public String debugString() { - StringBuilder out = new StringBuilder(); - out.append(Objects.toStringHelper(this) - .add("grouping_exprs", Expr.debugString(groupingExprs_)) - .add("aggregate_exprs", Expr.debugString(aggregateExprs_)) - .add("intermediate_tuple", (intermediateTupleDesc_ == null) - ? "null" : intermediateTupleDesc_.debugString()) - .add("output_tuple", (outputTupleDesc_ == null) - ? "null" : outputTupleDesc_.debugString()) - .toString()); - return out.toString(); - } - - protected abstract String tupleDebugName(); -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/AlterTableAddPartitionStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableAddPartitionStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/AlterTableAddPartitionStmt.java deleted file mode 100644 index a5cb2ca..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableAddPartitionStmt.java +++ /dev/null @@ -1,117 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.authorization.Privilege; -import com.cloudera.impala.catalog.HdfsTable; -import com.cloudera.impala.catalog.Table; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.FileSystemUtil; -import com.cloudera.impala.thrift.TAlterTableAddPartitionParams; -import com.cloudera.impala.thrift.TAlterTableParams; -import com.cloudera.impala.thrift.TAlterTableType; -import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.permission.FsAction; - -/** - * Represents an ALTER TABLE ADD PARTITION statement. - */ -public class AlterTableAddPartitionStmt extends AlterTableStmt { - private final HdfsUri location_; - private final boolean ifNotExists_; - private final PartitionSpec partitionSpec_; - private final HdfsCachingOp cacheOp_; - - public AlterTableAddPartitionStmt(TableName tableName, - PartitionSpec partitionSpec, HdfsUri location, boolean ifNotExists, - HdfsCachingOp cacheOp) { - super(tableName); - Preconditions.checkState(partitionSpec != null); - location_ = location; - ifNotExists_ = ifNotExists; - partitionSpec_ = partitionSpec; - partitionSpec_.setTableName(tableName); - cacheOp_ = cacheOp; - } - - public boolean getIfNotExists() { return ifNotExists_; } - public HdfsUri getLocation() { return location_; } - - @Override - public String toSql() { - StringBuilder sb = new StringBuilder("ALTER TABLE " + getTbl()); - sb.append(" ADD "); - if (ifNotExists_) { - sb.append("IF NOT EXISTS "); - } - sb.append(" " + partitionSpec_.toSql()); - if (location_ != null) { - sb.append(String.format(" LOCATION '%s'", location_)); - } - if (cacheOp_ != null) sb.append(cacheOp_.toSql()); - return sb.toString(); - } - - @Override - public TAlterTableParams toThrift() { - TAlterTableParams params = super.toThrift(); - params.setAlter_type(TAlterTableType.ADD_PARTITION); - TAlterTableAddPartitionParams addPartParams = new TAlterTableAddPartitionParams(); - addPartParams.setPartition_spec(partitionSpec_.toThrift()); - addPartParams.setLocation(location_ == null ? null : location_.toString()); - addPartParams.setIf_not_exists(ifNotExists_); - if (cacheOp_ != null) addPartParams.setCache_op(cacheOp_.toThrift()); - params.setAdd_partition_params(addPartParams); - return params; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - super.analyze(analyzer); - if (!ifNotExists_) partitionSpec_.setPartitionShouldNotExist(); - partitionSpec_.setPrivilegeRequirement(Privilege.ALTER); - partitionSpec_.analyze(analyzer); - - if (location_ != null) { - location_.analyze(analyzer, Privilege.ALL, FsAction.READ_WRITE); - } - - boolean shouldCache = false; - Table table = getTargetTable(); - if (cacheOp_ != null) { - cacheOp_.analyze(analyzer); - shouldCache = cacheOp_.shouldCache(); - } else if (table instanceof HdfsTable) { - shouldCache = ((HdfsTable)table).isMarkedCached(); - } - if (shouldCache) { - if (!(table instanceof HdfsTable)) { - throw new AnalysisException("Caching must target a HDFS table: " + - table.getFullName()); - } - HdfsTable hdfsTable = (HdfsTable)table; - if ((location_ != null && !FileSystemUtil.isPathCacheable(location_.getPath())) || - (location_ == null && !hdfsTable.isLocationCacheable())) { - throw new AnalysisException(String.format("Location '%s' cannot be cached. " + - "Please retry without caching: ALTER TABLE %s ADD PARTITION ... UNCACHED", - (location_ != null) ? location_.toString() : hdfsTable.getLocation(), - table.getFullName())); - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/AlterTableAddReplaceColsStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableAddReplaceColsStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/AlterTableAddReplaceColsStmt.java deleted file mode 100644 index aaa223a..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableAddReplaceColsStmt.java +++ /dev/null @@ -1,108 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.List; -import java.util.Set; - -import org.apache.hadoop.hive.metastore.api.FieldSchema; - -import com.cloudera.impala.catalog.Column; -import com.cloudera.impala.catalog.HBaseTable; -import com.cloudera.impala.catalog.Table; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TAlterTableAddReplaceColsParams; -import com.cloudera.impala.thrift.TAlterTableParams; -import com.cloudera.impala.thrift.TAlterTableType; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -/** - * Represents an ALTER TABLE ADD|REPLACE COLUMNS (colDef1, colDef2, ...) statement. - */ -public class AlterTableAddReplaceColsStmt extends AlterTableStmt { - private final List<ColumnDef> columnDefs_; - private final boolean replaceExistingCols_; - - public AlterTableAddReplaceColsStmt(TableName tableName, List<ColumnDef> columnDefs, - boolean replaceExistingCols) { - super(tableName); - Preconditions.checkState(columnDefs != null && columnDefs.size() > 0); - columnDefs_ = Lists.newArrayList(columnDefs); - replaceExistingCols_ = replaceExistingCols; - } - - public List<ColumnDef> getColumnDescs() { return columnDefs_; } - - // Replace columns instead of appending new columns. - public boolean getReplaceExistingCols() { - return replaceExistingCols_; - } - - @Override - public TAlterTableParams toThrift() { - TAlterTableParams params = super.toThrift(); - params.setAlter_type(TAlterTableType.ADD_REPLACE_COLUMNS); - TAlterTableAddReplaceColsParams colParams = new TAlterTableAddReplaceColsParams(); - for (ColumnDef col: getColumnDescs()) { - colParams.addToColumns(col.toThrift()); - } - colParams.setReplace_existing_cols(replaceExistingCols_); - params.setAdd_replace_cols_params(colParams); - return params; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - super.analyze(analyzer); - Table t = getTargetTable(); - // TODO: Support column-level DDL on HBase tables. Requires updating the column - // mappings along with the table columns. - if (t instanceof HBaseTable) { - throw new AnalysisException("ALTER TABLE ADD|REPLACE COLUMNS not currently " + - "supported on HBase tables."); - } - - // Build a set of the partition keys for the table. - Set<String> existingPartitionKeys = Sets.newHashSet(); - for (FieldSchema fs: t.getMetaStoreTable().getPartitionKeys()) { - existingPartitionKeys.add(fs.getName().toLowerCase()); - } - - // Make sure the new columns don't already exist in the table, that the names - // are all valid and unique, and that none of the columns conflict with - // partition columns. - Set<String> colNames = Sets.newHashSet(); - for (ColumnDef c: columnDefs_) { - c.analyze(); - String colName = c.getColName().toLowerCase(); - if (existingPartitionKeys.contains(colName)) { - throw new AnalysisException( - "Column name conflicts with existing partition column: " + colName); - } - - Column col = t.getColumn(colName); - if (col != null && !replaceExistingCols_) { - throw new AnalysisException("Column already exists: " + colName); - } else if (!colNames.add(colName)) { - throw new AnalysisException("Duplicate column name: " + colName); - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/AlterTableChangeColStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableChangeColStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/AlterTableChangeColStmt.java deleted file mode 100644 index c733ca0..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableChangeColStmt.java +++ /dev/null @@ -1,101 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import org.apache.hadoop.hive.metastore.api.FieldSchema; - -import com.cloudera.impala.catalog.HBaseTable; -import com.cloudera.impala.catalog.Table; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TAlterTableChangeColParams; -import com.cloudera.impala.thrift.TAlterTableParams; -import com.cloudera.impala.thrift.TAlterTableType; -import com.google.common.base.Preconditions; - -/** - * Represents an ALTER TABLE CHANGE COLUMN colName newColDef statement. - * Note: It would be fairly simple to reuse this class to support ALTER TABLE MODIFY - * newColDef statements in the future my making colName optional. - */ -public class AlterTableChangeColStmt extends AlterTableStmt { - private final String colName_; - private final ColumnDef newColDef_; - - public AlterTableChangeColStmt(TableName tableName, String colName, - ColumnDef newColDef) { - super(tableName); - Preconditions.checkNotNull(newColDef); - Preconditions.checkState(colName != null && !colName.isEmpty()); - colName_ = colName; - newColDef_ = newColDef; - } - - public String getColName() { return colName_; } - public ColumnDef getNewColDef() { return newColDef_; } - - @Override - public TAlterTableParams toThrift() { - TAlterTableParams params = super.toThrift(); - params.setAlter_type(TAlterTableType.CHANGE_COLUMN); - TAlterTableChangeColParams colParams = new TAlterTableChangeColParams(); - colParams.setCol_name(colName_); - colParams.setNew_col_def(newColDef_.toThrift()); - params.setChange_col_params(colParams); - return params; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - super.analyze(analyzer); - Table t = getTargetTable(); - // TODO: Support column-level DDL on HBase tables. Requires updating the column - // mappings along with the table columns. - if (t instanceof HBaseTable) { - throw new AnalysisException("ALTER TABLE CHANGE COLUMN not currently supported " + - "on HBase tables."); - } - String tableName = getDb() + "." + getTbl(); - - // Verify there are no conflicts with partition columns. - for (FieldSchema fs: t.getMetaStoreTable().getPartitionKeys()) { - if (fs.getName().toLowerCase().equals(colName_.toLowerCase())) { - throw new AnalysisException("Cannot modify partition column: " + colName_); - } - if (fs.getName().toLowerCase().equals(newColDef_.getColName().toLowerCase())) { - throw new AnalysisException( - "Column name conflicts with existing partition column: " + - newColDef_.getColName()); - } - } - - // Verify the column being modified exists in the table - if (t.getColumn(colName_) == null) { - throw new AnalysisException(String.format( - "Column '%s' does not exist in table: %s", colName_, tableName)); - } - - // Check that the new column def's name is valid. - newColDef_.analyze(); - // Verify that if the column name is being changed, the new name doesn't conflict - // with an existing column. - if (!colName_.toLowerCase().equals(newColDef_.getColName().toLowerCase()) && - t.getColumn(newColDef_.getColName()) != null) { - throw new AnalysisException("Column already exists: " + newColDef_.getColName()); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/AlterTableDropColStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableDropColStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/AlterTableDropColStmt.java deleted file mode 100644 index d7f5ab5..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableDropColStmt.java +++ /dev/null @@ -1,84 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import org.apache.hadoop.hive.metastore.api.FieldSchema; - -import com.cloudera.impala.catalog.HBaseTable; -import com.cloudera.impala.catalog.Table; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TAlterTableDropColParams; -import com.cloudera.impala.thrift.TAlterTableParams; -import com.cloudera.impala.thrift.TAlterTableType; -import com.google.common.base.Preconditions; - -/** - * Represents an ALTER TABLE DROP COLUMN statement. - * Note: Hive does not support this syntax for droppping columns, but it is supported - * by mysql. - */ -public class AlterTableDropColStmt extends AlterTableStmt { - private final String colName_; - - public AlterTableDropColStmt(TableName tableName, String colName) { - super(tableName); - Preconditions.checkState(colName != null && !colName.isEmpty()); - colName_ = colName; - } - - public String getColName() { return colName_; } - - @Override - public TAlterTableParams toThrift() { - TAlterTableParams params = super.toThrift(); - params.setAlter_type(TAlterTableType.DROP_COLUMN); - TAlterTableDropColParams dropColParams = new TAlterTableDropColParams(colName_); - params.setDrop_col_params(dropColParams); - return params; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - super.analyze(analyzer); - Table t = getTargetTable(); - // TODO: Support column-level DDL on HBase tables. Requires updating the column - // mappings along with the table columns. - if (t instanceof HBaseTable) { - throw new AnalysisException("ALTER TABLE DROP COLUMN not currently supported " + - "on HBase tables."); - } - String tableName = getDb() + "." + getTbl(); - - for (FieldSchema fs: t.getMetaStoreTable().getPartitionKeys()) { - if (fs.getName().toLowerCase().equals(colName_.toLowerCase())) { - throw new AnalysisException("Cannot drop partition column: " + fs.getName()); - } - } - - if (t.getColumns().size() - t.getMetaStoreTable().getPartitionKeysSize() <= 1) { - throw new AnalysisException(String.format( - "Cannot drop column '%s' from %s. Tables must contain at least 1 column.", - colName_, tableName)); - } - - if (t.getColumn(colName_) == null) { - throw new AnalysisException(String.format( - "Column '%s' does not exist in table: %s", colName_, tableName)); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/AlterTableDropPartitionStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableDropPartitionStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/AlterTableDropPartitionStmt.java deleted file mode 100644 index f8bc09c..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableDropPartitionStmt.java +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.authorization.Privilege; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TAlterTableDropPartitionParams; -import com.cloudera.impala.thrift.TAlterTableParams; -import com.cloudera.impala.thrift.TAlterTableType; -import com.google.common.base.Preconditions; - -/** - * Represents an ALTER TABLE DROP PARTITION statement. - */ -public class AlterTableDropPartitionStmt extends AlterTableStmt { - private final boolean ifExists_; - private final PartitionSpec partitionSpec_; - - // Setting this value causes dropped partition(s) to be permanently - // deleted. For example, for HDFS tables it skips the trash mechanism - private final boolean purgePartition_; - - public AlterTableDropPartitionStmt(TableName tableName, - PartitionSpec partitionSpec, boolean ifExists, boolean purgePartition) { - super(tableName); - Preconditions.checkNotNull(partitionSpec); - partitionSpec_ = partitionSpec; - partitionSpec_.setTableName(tableName); - ifExists_ = ifExists; - purgePartition_ = purgePartition; - } - - public boolean getIfNotExists() { return ifExists_; } - - @Override - public String toSql() { - StringBuilder sb = new StringBuilder("ALTER TABLE " + getTbl()); - sb.append(" DROP "); - if (ifExists_) sb.append("IF EXISTS "); - sb.append(" DROP " + partitionSpec_.toSql()); - if (purgePartition_) sb.append(" PURGE"); - return sb.toString(); - } - - @Override - public TAlterTableParams toThrift() { - TAlterTableParams params = super.toThrift(); - params.setAlter_type(TAlterTableType.DROP_PARTITION); - TAlterTableDropPartitionParams addPartParams = new TAlterTableDropPartitionParams(); - addPartParams.setPartition_spec(partitionSpec_.toThrift()); - addPartParams.setIf_exists(ifExists_); - addPartParams.setPurge(purgePartition_); - params.setDrop_partition_params(addPartParams); - return params; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - super.analyze(analyzer); - if (!ifExists_) partitionSpec_.setPartitionShouldExist(); - partitionSpec_.setPrivilegeRequirement(Privilege.ALTER); - partitionSpec_.analyze(analyzer); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/AlterTableOrViewRenameStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableOrViewRenameStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/AlterTableOrViewRenameStmt.java deleted file mode 100644 index 009535c..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableOrViewRenameStmt.java +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.authorization.Privilege; -import com.cloudera.impala.catalog.View; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TAccessEvent; -import com.cloudera.impala.thrift.TAlterTableOrViewRenameParams; -import com.cloudera.impala.thrift.TAlterTableParams; -import com.cloudera.impala.thrift.TAlterTableType; -import com.cloudera.impala.thrift.TTableName; -import com.google.common.base.Preconditions; - -/** - * Represents an ALTER TABLE/VIEW RENAME statement. - */ -public class AlterTableOrViewRenameStmt extends AlterTableStmt { - protected final TableName newTableName_; - - // Set during analysis - protected String newDbName_; - - // True if we are renaming a table. False if we are renaming a view. - protected final boolean renameTable_; - - public AlterTableOrViewRenameStmt(TableName oldTableName, TableName newTableName, - boolean renameTable) { - super(oldTableName); - Preconditions.checkState(newTableName != null && !newTableName.isEmpty()); - newTableName_ = newTableName; - renameTable_ = renameTable; - } - - public String getNewTbl() { - return newTableName_.getTbl(); - } - - public String getNewDb() { - Preconditions.checkNotNull(newDbName_); - return newDbName_; - } - - @Override - public TAlterTableParams toThrift() { - TAlterTableParams params = super.toThrift(); - params.setAlter_type( - (renameTable_) ? TAlterTableType.RENAME_TABLE : TAlterTableType.RENAME_VIEW); - TAlterTableOrViewRenameParams renameParams = - new TAlterTableOrViewRenameParams(new TTableName(getNewDb(), getNewTbl())); - params.setRename_params(renameParams); - return params; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - newTableName_.analyze(); - table_ = analyzer.getTable(tableName_, Privilege.ALTER); - if (table_ instanceof View && renameTable_) { - throw new AnalysisException(String.format( - "ALTER TABLE not allowed on a view: %s", table_.getFullName())); - } - if (!(table_ instanceof View) && !renameTable_) { - throw new AnalysisException(String.format( - "ALTER VIEW not allowed on a table: %s", table_.getFullName())); - } - newDbName_ = analyzer.getTargetDbName(newTableName_); - if (analyzer.dbContainsTable(newDbName_, newTableName_.getTbl(), Privilege.CREATE)) { - throw new AnalysisException(Analyzer.TBL_ALREADY_EXISTS_ERROR_MSG + - String.format("%s.%s", newDbName_, getNewTbl())); - } - analyzer.addAccessEvent(new TAccessEvent(newDbName_ + "." + newTableName_.getTbl(), - table_.getCatalogObjectType(), Privilege.CREATE.toString())); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/AlterTableRecoverPartitionsStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableRecoverPartitionsStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/AlterTableRecoverPartitionsStmt.java deleted file mode 100644 index c7e796c..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/AlterTableRecoverPartitionsStmt.java +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.catalog.HdfsTable; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TAlterTableParams; -import com.cloudera.impala.thrift.TAlterTableType; - -/** - * Represents an ALTER TABLE RECOVER PARTITIONS statement. - */ -public class AlterTableRecoverPartitionsStmt extends AlterTableStmt { - - public AlterTableRecoverPartitionsStmt(TableName tableName) { - super(tableName); - } - - @Override - public TAlterTableParams toThrift() { - TAlterTableParams params = super.toThrift(); - params.setAlter_type(TAlterTableType.RECOVER_PARTITIONS); - return params; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - super.analyze(analyzer); - - // Make sure the target table is HdfsTable. - if (!(table_ instanceof HdfsTable)) { - throw new AnalysisException("ALTER TABLE RECOVER PARTITIONS " + - "must target an HDFS table: " + tableName_); - } - - // Make sure the target table is partitioned. - if (table_.getMetaStoreTable().getPartitionKeysSize() == 0) { - throw new AnalysisException("Table is not partitioned: " + tableName_); - } - } -}
