http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java b/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java new file mode 100644 index 0000000..b1f9b95 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import java.util.ArrayList; + +import org.apache.commons.lang3.StringUtils; + +import parquet.Strings; + +import com.cloudera.impala.analysis.Path.PathType; +import com.cloudera.impala.authorization.Privilege; +import com.cloudera.impala.authorization.PrivilegeRequestBuilder; +import com.cloudera.impala.catalog.StructType; +import com.cloudera.impala.catalog.TableLoadingException; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TDescribeOutputStyle; +import com.cloudera.impala.thrift.TDescribeTableParams; +import com.google.common.base.Preconditions; + +/** + * Representation of a DESCRIBE table statement which returns metadata on + * a specified table: + * Syntax: DESCRIBE <path> + * DESCRIBE FORMATTED|EXTENDED <table> + * + * If FORMATTED|EXTENDED is not specified and the path refers to a table, the statement + * only returns info on the given table's column definition (column name, data type, and + * comment). If the path refers to a complex typed field within a column, the statement + * returns the field names, types, and comments. + * If FORMATTED|EXTENDED is specified, extended metadata on the table is returned + * (in addition to the column definitions). This metadata includes info about the table + * properties, SerDe properties, StorageDescriptor properties, and more. + */ +public class DescribeTableStmt extends StatementBase { + private final TDescribeOutputStyle outputStyle_; + + /// "."-separated path from the describe statement. + private ArrayList<String> rawPath_; + + /// The resolved path to describe, set after analysis. + private Path path_; + + /// The fully qualified name of the root table, set after analysis. + private TableName tableName_; + + /// Struct type with the fields to display for the described path. + private StructType resultStruct_; + + public DescribeTableStmt(ArrayList<String> rawPath, TDescribeOutputStyle outputStyle) { + Preconditions.checkNotNull(rawPath); + Preconditions.checkArgument(!rawPath.isEmpty()); + rawPath_ = rawPath; + outputStyle_ = outputStyle; + path_ = null; + tableName_ = null; + resultStruct_ = null; + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder("DESCRIBE "); + if (outputStyle_ != TDescribeOutputStyle.MINIMAL) { + sb.append(outputStyle_.toString() + " "); + } + return sb.toString() + StringUtils.join(rawPath_, "."); + } + + public TableName getTableName() { return tableName_; } + public TDescribeOutputStyle getOutputStyle() { return outputStyle_; } + + + /** + * Get the privilege requirement, which depends on the output style. + */ + private Privilege getPrivilegeRequirement() { + switch (outputStyle_) { + case MINIMAL: return Privilege.ANY; + case FORMATTED: + case EXTENDED: + return Privilege.VIEW_METADATA; + default: + Preconditions.checkArgument(false); + return null; + } + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + try { + path_ = analyzer.resolvePath(rawPath_, PathType.ANY); + } catch (AnalysisException ae) { + // Register privilege requests to prefer reporting an authorization error over + // an analysis error. We should not accidentally reveal the non-existence of a + // table/database if the user is not authorized. + if (analyzer.hasMissingTbls()) throw ae; + if (rawPath_.size() > 1) { + analyzer.registerPrivReq(new PrivilegeRequestBuilder() + .onTable(rawPath_.get(0), rawPath_.get(1)) + .allOf(getPrivilegeRequirement()).toRequest()); + } + analyzer.registerPrivReq(new PrivilegeRequestBuilder() + .onTable(analyzer.getDefaultDb(), rawPath_.get(0)) + .allOf(getPrivilegeRequirement()).toRequest()); + throw ae; + } catch (TableLoadingException tle) { + throw new AnalysisException(tle.getMessage(), tle); + } + + tableName_ = analyzer.getFqTableName(path_.getRootTable().getTableName()); + analyzer.getTable(tableName_, getPrivilegeRequirement()); + + if (path_.destTable() != null) { + resultStruct_ = path_.getRootTable().getHiveColumnsAsStruct(); + } else if (path_.destType().isComplexType()) { + if (outputStyle_ == TDescribeOutputStyle.FORMATTED || + outputStyle_ == TDescribeOutputStyle.EXTENDED) { + throw new AnalysisException("DESCRIBE FORMATTED|EXTENDED must refer to a table"); + } + Preconditions.checkState(outputStyle_ == TDescribeOutputStyle.MINIMAL); + resultStruct_ = Path.getTypeAsStruct(path_.destType()); + } else { + throw new AnalysisException("Cannot describe path '" + + Strings.join(rawPath_, ".") + "' targeting scalar type: " + + path_.destType().toSql()); + } + } + + public TDescribeTableParams toThrift() { + TDescribeTableParams params = new TDescribeTableParams(); + params.setTable_name(getTableName().getTbl()); + params.setDb(getTableName().getDb()); + params.setOutput_style(outputStyle_); + params.setResult_struct(resultStruct_.toThrift()); + return params; + } +}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java b/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java new file mode 100644 index 0000000..c0d7571 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +import org.apache.commons.lang.StringUtils; + +import com.cloudera.impala.catalog.Table; +import com.cloudera.impala.catalog.View; +import com.cloudera.impala.common.IdGenerator; +import com.cloudera.impala.thrift.TDescriptorTable; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +/** + * Repository for tuple (and slot) descriptors. + * Descriptors should only be created through this class, which assigns + * them unique ids. + */ +public class DescriptorTable { + private final HashMap<TupleId, TupleDescriptor> tupleDescs_ = Maps.newHashMap(); + private final HashMap<SlotId, SlotDescriptor> slotDescs_ = Maps.newHashMap(); + private final IdGenerator<TupleId> tupleIdGenerator_ = TupleId.createGenerator(); + private final IdGenerator<SlotId> slotIdGenerator_ = SlotId.createGenerator(); + // List of referenced tables with no associated TupleDescriptor to ship to the BE. + // For example, the output table of an insert query. + private final List<Table> referencedTables_ = Lists.newArrayList(); + // For each table, the set of partitions that are referenced by at least one scan range. + private final HashMap<Table, HashSet<Long>> referencedPartitionsPerTable_ = + Maps.newHashMap(); + + public TupleDescriptor createTupleDescriptor(String debugName) { + TupleDescriptor d = new TupleDescriptor(tupleIdGenerator_.getNextId(), debugName); + tupleDescs_.put(d.getId(), d); + return d; + } + + /** + * Create copy of src with new id. The returned descriptor has its mem layout + * computed. + */ + public TupleDescriptor copyTupleDescriptor(TupleId srcId, String debugName) { + TupleDescriptor d = new TupleDescriptor(tupleIdGenerator_.getNextId(), debugName); + tupleDescs_.put(d.getId(), d); + // create copies of slots + TupleDescriptor src = tupleDescs_.get(srcId); + for (SlotDescriptor slot: src.getSlots()) { + copySlotDescriptor(d, slot); + } + d.computeMemLayout(); + Preconditions.checkState(d.getByteSize() == src.getByteSize()); + return d; + } + + public SlotDescriptor addSlotDescriptor(TupleDescriptor d) { + SlotDescriptor result = new SlotDescriptor(slotIdGenerator_.getNextId(), d); + d.addSlot(result); + slotDescs_.put(result.getId(), result); + return result; + } + + /** + * Append copy of src to dest. + */ + public SlotDescriptor copySlotDescriptor(TupleDescriptor dest, SlotDescriptor src) { + SlotDescriptor result = new SlotDescriptor(slotIdGenerator_.getNextId(), dest, src); + dest.addSlot(result); + slotDescs_.put(result.getId(), result); + return result; + } + + public TupleDescriptor getTupleDesc(TupleId id) { return tupleDescs_.get(id); } + public SlotDescriptor getSlotDesc(SlotId id) { return slotDescs_.get(id); } + public Collection<TupleDescriptor> getTupleDescs() { return tupleDescs_.values(); } + public Collection<SlotDescriptor> getSlotDescs() { return slotDescs_.values(); } + public TupleId getMaxTupleId() { return tupleIdGenerator_.getMaxId(); } + public SlotId getMaxSlotId() { return slotIdGenerator_.getMaxId(); } + + public void addReferencedTable(Table table) { + referencedTables_.add(table); + } + + /** + * Find the set of referenced partitions for the given table. Allocates a set if + * none has been allocated for the table yet. + */ + private HashSet<Long> getReferencedPartitions(Table table) { + HashSet<Long> refPartitions = referencedPartitionsPerTable_.get(table); + if (refPartitions == null) { + refPartitions = new HashSet<Long>(); + referencedPartitionsPerTable_.put(table, refPartitions); + } + return refPartitions; + } + + /** + * Add the partition with ID partitionId to the set of referenced partitions for the + * given table. + */ + public void addReferencedPartition(Table table, long partitionId) { + getReferencedPartitions(table).add(partitionId); + } + + /** + * Marks all slots in list as materialized. + */ + public void markSlotsMaterialized(List<SlotId> ids) { + for (SlotId id: ids) { + getSlotDesc(id).setIsMaterialized(true); + } + } + + /** + * Return all ids in slotIds that belong to tupleId. + */ + public List<SlotId> getTupleSlotIds(List<SlotId> slotIds, TupleId tupleId) { + List<SlotId> result = Lists.newArrayList(); + for (SlotId id: slotIds) { + if (getSlotDesc(id).getParent().getId().equals(tupleId)) result.add(id); + } + return result; + } + + // Computes physical layout parameters of all descriptors. + // Call this only after the last descriptor was added. + // Test-only. + public void computeMemLayout() { + for (TupleDescriptor d: tupleDescs_.values()) { + d.computeMemLayout(); + } + } + + public TDescriptorTable toThrift() { + TDescriptorTable result = new TDescriptorTable(); + HashSet<Table> referencedTbls = Sets.newHashSet(); + HashSet<Table> allPartitionsTbls = Sets.newHashSet(); + for (TupleDescriptor tupleDesc: tupleDescs_.values()) { + // inline view of a non-constant select has a non-materialized tuple descriptor + // in the descriptor table just for type checking, which we need to skip + if (tupleDesc.isMaterialized()) { + // TODO: Ideally, we should call tupleDesc.checkIsExecutable() here, but there + // currently are several situations in which we send materialized tuples without + // a mem layout to the BE, e.g., when unnesting unions or when replacing plan + // trees with an EmptySetNode. + result.addToTupleDescriptors(tupleDesc.toThrift()); + Table table = tupleDesc.getTable(); + if (table != null && !(table instanceof View)) referencedTbls.add(table); + // Only serialize materialized slots + for (SlotDescriptor slotD: tupleDesc.getMaterializedSlots()) { + result.addToSlotDescriptors(slotD.toThrift()); + } + } + } + for (Table table: referencedTables_) { + referencedTbls.add(table); + // We don't know which partitions are needed for INSERT, so include them all. + allPartitionsTbls.add(table); + } + for (Table tbl: referencedTbls) { + HashSet<Long> referencedPartitions = null; // null means include all partitions. + if (!allPartitionsTbls.contains(tbl)) { + referencedPartitions = getReferencedPartitions(tbl); + } + result.addToTableDescriptors(tbl.toThriftDescriptor(referencedPartitions)); + } + return result; + } + + public String debugString() { + StringBuilder out = new StringBuilder(); + out.append("tuples:\n"); + for (TupleDescriptor desc: tupleDescs_.values()) { + out.append(desc.debugString() + "\n"); + } + return out.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/DistributeParam.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/DistributeParam.java b/fe/src/main/java/org/apache/impala/analysis/DistributeParam.java new file mode 100644 index 0000000..e718d6b --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/DistributeParam.java @@ -0,0 +1,199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TDistributeByHashParam; +import com.cloudera.impala.thrift.TDistributeByRangeParam; +import com.cloudera.impala.thrift.TDistributeParam; +import com.cloudera.impala.thrift.TDistributeType; +import com.cloudera.impala.thrift.TRangeLiteral; +import com.cloudera.impala.thrift.TRangeLiteralList; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +/** + * Represents the information of + * + * DISTRIBUTE BY HASH[(col_def_list)] INTO n BUCKETS + * DISTRIBUTE BY RANGE[(col_def_list)] SPLIT ROWS ( (v1,v2,v3), ...) + * + * clauses in CREATE TABLE statements, where available, e.g. Kudu. + * + * A table can be hash or range partitioned, or combinations of both. A distribute + * clause represents one particular distribution rule. For both HASH and RANGE types, + * some of the error checking is done during the analysis, but most of it is deferred + * until the table is actually created. + */ +public class DistributeParam implements ParseNode { + + /** + * Creates a DistributeParam partitioned by hash. + */ + public static DistributeParam createHashParam(List<String> cols, BigDecimal buckets) { + return new DistributeParam(Type.HASH, cols, buckets); + } + + /** + * Creates a DistributeParam partitioned by range. + */ + public static DistributeParam createRangeParam(List<String> cols, + ArrayList<ArrayList<LiteralExpr>> splitRows) { + return new DistributeParam(Type.RANGE, cols, splitRows); + } + + private static final int NO_BUCKETS = -1; + + /** + * The type of the distribution rule. + */ + public enum Type { + HASH, RANGE + }; + + private List<String> columns_; + + private final Type type_; + + // Only relevant for hash partitioning, -1 otherwise + private final int num_buckets_; + + // Only relevant for range partitioning, null otherwise + private final ArrayList<ArrayList<LiteralExpr>> splitRows_; + + // Set in analyze() + private TDistributeByRangeParam rangeParam_; + + private DistributeParam(Type t, List<String> cols, BigDecimal buckets) { + type_ = t; + columns_ = cols; + num_buckets_ = buckets.intValue(); + splitRows_ = null; + } + + private DistributeParam(Type t, List<String> cols, + ArrayList<ArrayList<LiteralExpr>> splitRows) { + type_ = t; + columns_ = cols; + splitRows_ = splitRows; + num_buckets_ = NO_BUCKETS; + } + + /** + * TODO Refactor the logic below to analyze 'columns_'. This analysis should output + * a vector of column types that would then be used during the analysis of the split + * rows. + */ + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (type_ == Type.HASH && num_buckets_ <= 1) { + throw new AnalysisException(String.format( + "Number of buckets in DISTRIBUTE BY clause '%s' must be larger than 1.", + toSql())); + } else if (type_ == Type.RANGE) { + // Creating the thrift structure simultaneously checks for semantic errors + rangeParam_ = new TDistributeByRangeParam(); + rangeParam_.setColumns(columns_); + + for (ArrayList<LiteralExpr> splitRow : splitRows_) { + TRangeLiteralList list = new TRangeLiteralList(); + if (splitRow.size() != columns_.size()) { + throw new AnalysisException(String.format( + "SPLIT ROWS has different size than number of projected key columns: %d. " + + "Split row: %s", columns_.size(), splitRowToString(splitRow))); + } + for (LiteralExpr expr : splitRow) { + expr.analyze(analyzer); + TRangeLiteral literal = new TRangeLiteral(); + if (expr instanceof NumericLiteral) { + NumericLiteral num = (NumericLiteral) expr; + if (num.getType().isDecimal() || num.getType().isFloatingPointType()) { + throw new AnalysisException("Only integral and string values allowed for" + + " split rows."); + } else { + literal.setInt_literal(num.getIntValue()); + } + } else if (expr instanceof StringLiteral) { + StringLiteral string = (StringLiteral) expr; + literal.setString_literal(string.getStringValue()); + } else if (expr instanceof BoolLiteral) { + BoolLiteral bool = (BoolLiteral) expr; + literal.setBool_literal(bool.getValue()); + } else { + throw new AnalysisException(String.format("Split row value is not supported: " + + "%s (Type: %s).", expr.getStringValue(), expr.getType().toSql())); + } + list.addToValues(literal); + } + rangeParam_.addToSplit_rows(list); + } + } + } + + @Override + public String toSql() { + if (num_buckets_ == NO_BUCKETS) { + List<String> splitRowStrings = Lists.newArrayList(); + for (ArrayList<LiteralExpr> splitRow : splitRows_) { + splitRowStrings.add(splitRowToString(splitRow)); + } + return String.format("RANGE(%s) INTO RANGES(%s)", Joiner.on(", ").join(columns_), + Joiner.on(", ").join(splitRowStrings)); + } else { + return String.format("HASH(%s) INTO %d BUCKETS", Joiner.on(", ").join(columns_), + num_buckets_); + } + } + + private String splitRowToString(ArrayList<LiteralExpr> splitRow) { + StringBuilder builder = new StringBuilder(); + builder.append("("); + List<String> rangeElementStrings = Lists.newArrayList(); + for (LiteralExpr rangeElement : splitRow) { + rangeElementStrings.add(rangeElement.toSql()); + } + builder.append(Joiner.on(", ").join(rangeElementStrings)); + builder.append(")"); + return builder.toString(); + } + + TDistributeParam toThrift() { + TDistributeParam result = new TDistributeParam(); + if (type_ == Type.HASH) { + TDistributeByHashParam hash = new TDistributeByHashParam(); + hash.setNum_buckets(num_buckets_); + hash.setColumns(columns_); + result.setBy_hash_param(hash); + } else { + Preconditions.checkState(type_ == Type.RANGE); + + result.setBy_range_param(rangeParam_); + } + return result; + } + + public List<String> getColumns() { return columns_; } + public void setColumns(List<String> cols) { columns_ = cols; } + public Type getType_() { return type_; } + public int getNumBuckets() { return num_buckets_; } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/DropDataSrcStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/DropDataSrcStmt.java b/fe/src/main/java/org/apache/impala/analysis/DropDataSrcStmt.java new file mode 100644 index 0000000..f5642fa --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/DropDataSrcStmt.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import org.apache.hadoop.hive.metastore.MetaStoreUtils; + +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TDropDataSourceParams; +import com.google.common.base.Preconditions; + +/** + * Represents a DROP DATA SOURCE statement. + */ +public class DropDataSrcStmt extends StatementBase { + + private final String dataSrcName_; + private final boolean ifExists_; + + public DropDataSrcStmt(String dataSrcName, boolean ifExists) { + Preconditions.checkNotNull(dataSrcName); + this.dataSrcName_ = dataSrcName.toLowerCase(); + this.ifExists_ = ifExists; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (!MetaStoreUtils.validateName(dataSrcName_) || + (!ifExists_ && analyzer.getCatalog().getDataSource(dataSrcName_) == null)) { + throw new AnalysisException(Analyzer.DATA_SRC_DOES_NOT_EXIST_ERROR_MSG + + dataSrcName_); + } + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("DROP DATA SOURCE "); + if (ifExists_) sb.append("IF EXISTS "); + sb.append(dataSrcName_); + return sb.toString(); + } + + public TDropDataSourceParams toThrift() { + return new TDropDataSourceParams(dataSrcName_).setIf_exists(ifExists_); + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/DropDbStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/DropDbStmt.java b/fe/src/main/java/org/apache/impala/analysis/DropDbStmt.java new file mode 100644 index 0000000..af7fae1 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/DropDbStmt.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.authorization.Privilege; +import com.cloudera.impala.catalog.Db; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TDropDbParams; + +/** + * Represents a DROP [IF EXISTS] DATABASE [CASCADE | RESTRICT] statement + */ +public class DropDbStmt extends StatementBase { + private final String dbName_; + private final boolean ifExists_; + private final boolean cascade_; + + /** + * Constructor for building the drop statement. If ifExists is true, an error will not + * be thrown if the database does not exist. If cascade is true, all the tables in the + * database will be dropped. + */ + public DropDbStmt(String dbName, boolean ifExists, boolean cascade) { + this.dbName_ = dbName; + this.ifExists_ = ifExists; + this.cascade_ = cascade; + } + + public String getDb() { return dbName_; } + public boolean getIfExists() { return ifExists_; } + public boolean getCascade() { return cascade_; } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder("DROP DATABASE"); + if (ifExists_) sb.append(" IF EXISTS "); + sb.append(getDb()); + if (cascade_) sb.append(" CASCADE"); + return sb.toString(); + } + + public TDropDbParams toThrift() { + TDropDbParams params = new TDropDbParams(); + params.setDb(getDb()); + params.setIf_exists(getIfExists()); + params.setCascade(getCascade()); + return params; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + Db db = analyzer.getDb(dbName_, Privilege.DROP, false); + if (db == null && !ifExists_) { + throw new AnalysisException(Analyzer.DB_DOES_NOT_EXIST_ERROR_MSG + dbName_); + } + + if (analyzer.getDefaultDb().toLowerCase().equals(dbName_.toLowerCase())) { + throw new AnalysisException("Cannot drop current default database: " + dbName_); + } + if (db != null && db.numFunctions() > 0 && !cascade_) { + throw new AnalysisException("Cannot drop non-empty database: " + dbName_); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/DropFunctionStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/DropFunctionStmt.java b/fe/src/main/java/org/apache/impala/analysis/DropFunctionStmt.java new file mode 100644 index 0000000..39f5ff9 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/DropFunctionStmt.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.authorization.AuthorizeableFn; +import com.cloudera.impala.authorization.Privilege; +import com.cloudera.impala.authorization.PrivilegeRequest; +import com.cloudera.impala.catalog.Db; +import com.cloudera.impala.catalog.Function; +import com.cloudera.impala.catalog.Type; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TDropFunctionParams; +import com.cloudera.impala.thrift.TFunctionCategory; + +import java.util.ArrayList; + +/** + * Represents a DROP [IF EXISTS] FUNCTION statement + * TODO: try to consolidate this with the other Drop*Stmt class, perhaps + * by adding a DropStatementBase class. + */ +public class DropFunctionStmt extends StatementBase { + private final FunctionName fnName_; + private final FunctionArgs fnArgs_; + private final boolean ifExists_; + + // Set in analyze(). + private Function desc_; + + /** + * Constructor for building the drop statement. If ifExists is true, an error will not + * be thrown if the function does not exist. + */ + public DropFunctionStmt(FunctionName fnName, FunctionArgs fnArgs, boolean ifExists) { + fnName_ = fnName; + fnArgs_ = fnArgs; + ifExists_ = ifExists; + } + + public FunctionName getFunction() { return desc_.getFunctionName(); } + public boolean getIfExists() { return ifExists_; } + private boolean hasSignature() { return fnArgs_ != null; } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder("DROP FUNCTION"); + if (ifExists_) sb.append(" IF EXISTS "); + sb.append(desc_.signatureString()); + sb.append(")"); + return sb.toString(); + } + + public TDropFunctionParams toThrift() { + TDropFunctionParams params = new TDropFunctionParams(); + params.setFn_name(desc_.getFunctionName().toThrift()); + params.setArg_types(Type.toThrift(desc_.getArgs())); + params.setIf_exists(getIfExists()); + if (hasSignature()) params.setSignature(desc_.signatureString()); + return params; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + fnName_.analyze(analyzer); + + if (hasSignature()) { + fnArgs_.analyze(analyzer); + desc_ = new Function(fnName_, fnArgs_.getArgTypes(), Type.INVALID, + fnArgs_.hasVarArgs()); + } else { + desc_ = new Function(fnName_, new ArrayList<Type>(), Type.INVALID, + false); + } + + // For now, if authorization is enabled, the user needs ALL on the server + // to drop functions. + // TODO: this is not the right granularity but acceptable for now. + analyzer.registerPrivReq(new PrivilegeRequest( + new AuthorizeableFn(desc_.signatureString()), Privilege.ALL)); + + Db db = analyzer.getDb(desc_.dbName(), Privilege.DROP, false); + if (db == null && !ifExists_) { + throw new AnalysisException(Analyzer.DB_DOES_NOT_EXIST_ERROR_MSG + desc_.dbName()); + } + + if (!hasSignature() && db != null && db.getFunctions( + desc_.functionName()).isEmpty() && !ifExists_) { + throw new AnalysisException( + Analyzer.FN_DOES_NOT_EXIST_ERROR_MSG + desc_.functionName()); + } + + if (hasSignature() && analyzer.getCatalog().getFunction( + desc_, Function.CompareMode.IS_IDENTICAL) == null && !ifExists_) { + throw new AnalysisException( + Analyzer.FN_DOES_NOT_EXIST_ERROR_MSG + desc_.signatureString()); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java b/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java new file mode 100644 index 0000000..90f9434 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.authorization.Privilege; +import com.cloudera.impala.catalog.Table; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TDropStatsParams; +import com.cloudera.impala.thrift.TTableName; +import com.google.common.base.Preconditions; + +/** + * Represents both a DROP STATS statement, and the DROP INCREMENTAL STATS <tbl> PARTITION + * <part_spec> variant. + */ +public class DropStatsStmt extends StatementBase { + protected final TableName tableName_; + + // If non-null, only drop the statistics for a given partition + PartitionSpec partitionSpec_ = null; + + // Set during analysis + protected String dbName_; + + /** + * Constructor for building the DROP TABLE/VIEW statement + */ + public DropStatsStmt(TableName tableName) { + this.tableName_ = tableName; + } + + public DropStatsStmt(TableName tableName, PartitionSpec partSpec) { + this.tableName_ = tableName; + this.partitionSpec_ = partSpec; + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder("DROP "); + if (partitionSpec_ == null) { + sb.append(" STATS "); + if (tableName_.getDb() != null) sb.append(tableName_.getDb() + "."); + sb.append(tableName_.toSql()); + } else { + sb.append(" INCREMENTAL STATS "); + if (tableName_.getDb() != null) sb.append(tableName_.getDb() + "."); + sb.append(tableName_.toSql()); + sb.append(partitionSpec_.toSql()); + } + return sb.toString(); + } + + public TDropStatsParams toThrift() { + TDropStatsParams params = new TDropStatsParams(); + params.setTable_name(new TTableName(getDb(), getTbl())); + + if (partitionSpec_ != null) { + params.setPartition_spec(partitionSpec_.toThrift()); + } + return params; + } + + /** + * Checks that the given table exists and the user has privileges + * to drop stats on this table. + */ + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + dbName_ = analyzer.getTargetDbName(tableName_); + Table table = analyzer.getTable(tableName_, Privilege.ALTER); + Preconditions.checkNotNull(table); + if (partitionSpec_ != null) { + partitionSpec_.setTableName(tableName_); + partitionSpec_.setPrivilegeRequirement(Privilege.ALTER); + partitionSpec_.setPartitionShouldExist(); + partitionSpec_.analyze(analyzer); + } + } + + /** + * Can only be called after analysis. Returns the name of the database that + * the target drop table resides in. + */ + public String getDb() { + Preconditions.checkNotNull(dbName_); + return dbName_; + } + + public String getTbl() { return tableName_.getTbl(); } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java b/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java new file mode 100644 index 0000000..8371ace --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.authorization.Privilege; +import com.cloudera.impala.catalog.Table; +import com.cloudera.impala.catalog.View; +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TDropTableOrViewParams; +import com.cloudera.impala.thrift.TTableName; +import com.google.common.base.Preconditions; + +/** + * Represents a DROP TABLE/VIEW [IF EXISTS] statement + */ +public class DropTableOrViewStmt extends StatementBase { + protected final TableName tableName_; + protected final boolean ifExists_; + + // True if we are dropping a table. False if we are dropping a view. + protected final boolean dropTable_; + + // Setting this value causes dropped tables to be permanently + // deleted. For example, for hdfs tables it skips the trash directory + protected final boolean purgeTable_; + + // Set during analysis + protected String dbName_; + + /** + * Constructor for building the DROP TABLE/VIEW statement + */ + public DropTableOrViewStmt(TableName tableName, boolean ifExists, + boolean dropTable, boolean purgeTable) { + tableName_ = tableName; + ifExists_ = ifExists; + dropTable_ = dropTable; + purgeTable_ = purgeTable; + // PURGE with a view is not allowed. + Preconditions.checkState(!(!dropTable_ && purgeTable_)); + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder("DROP " + ((dropTable_) ? "TABLE " : "VIEW ")); + if (ifExists_) sb.append("IF EXISTS "); + if (tableName_.getDb() != null) sb.append(tableName_.getDb() + "."); + sb.append(tableName_.getTbl()); + if (purgeTable_) sb.append(" PURGE"); + return sb.toString(); + } + + public TDropTableOrViewParams toThrift() { + TDropTableOrViewParams params = new TDropTableOrViewParams(); + params.setTable_name(new TTableName(getDb(), getTbl())); + params.setIf_exists(ifExists_); + params.setPurge(purgeTable_); + params.setIs_table(dropTable_); + return params; + } + + /** + * 1. Checks that the user has privileges to DROP the given table/view + * 2. Checks that the database and table exists + * 3. Checks that the table type (TABLE/VIEW) matches the DROP TABLE/VIEW statement + * Note: Do not analyze tableName because we prefer to report an error indicating + * that the table/view does not exist even if the table/view name is invalid. + */ + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + dbName_ = analyzer.getTargetDbName(tableName_); + try { + Table table = analyzer.getTable(tableName_, Privilege.DROP); + Preconditions.checkNotNull(table); + if (table instanceof View && dropTable_) { + throw new AnalysisException(String.format( + "DROP TABLE not allowed on a view: %s.%s", dbName_, getTbl())); + } + if (!(table instanceof View) && !dropTable_) { + throw new AnalysisException(String.format( + "DROP VIEW not allowed on a table: %s.%s", dbName_, getTbl())); + } + } catch (AnalysisException e) { + if (ifExists_ && analyzer.getMissingTbls().isEmpty()) return; + throw e; + } + } + + /** + * Can only be called after analysis. Returns the name of the database that + * the target drop table resides in. + */ + public String getDb() { + Preconditions.checkNotNull(dbName_); + return dbName_; + } + + public String getTbl() { return tableName_.getTbl(); } + public boolean isDropTable() { return dropTable_; } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/EquivalenceClassId.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/EquivalenceClassId.java b/fe/src/main/java/org/apache/impala/analysis/EquivalenceClassId.java new file mode 100644 index 0000000..df658b9 --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/EquivalenceClassId.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.common.Id; +import com.cloudera.impala.common.IdGenerator; + +public class EquivalenceClassId extends Id<EquivalenceClassId> { + // Construction only allowed via an IdGenerator. + protected EquivalenceClassId(int id) { + super(id); + } + + public static IdGenerator<EquivalenceClassId> createGenerator() { + return new IdGenerator<EquivalenceClassId>() { + @Override + public EquivalenceClassId getNextId() { return new EquivalenceClassId(nextId_++); } + @Override + public EquivalenceClassId getMaxId() { return new EquivalenceClassId(nextId_ - 1); } + }; + } +} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/org/apache/impala/analysis/ExistsPredicate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/ExistsPredicate.java b/fe/src/main/java/org/apache/impala/analysis/ExistsPredicate.java new file mode 100644 index 0000000..da984eb --- /dev/null +++ b/fe/src/main/java/org/apache/impala/analysis/ExistsPredicate.java @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloudera.impala.analysis; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cloudera.impala.common.AnalysisException; +import com.cloudera.impala.thrift.TExprNode; +import com.google.common.base.Preconditions; + +/** + * Class representing a [NOT] EXISTS predicate. + */ +public class ExistsPredicate extends Predicate { + private final static Logger LOG = LoggerFactory.getLogger( + ExistsPredicate.class); + private boolean notExists_ = false; + + public boolean isNotExists() { return notExists_; } + + /** + * C'tor that initializes an ExistsPredicate from a Subquery. + */ + public ExistsPredicate(Subquery subquery, boolean notExists) { + Preconditions.checkNotNull(subquery); + children_.add(subquery); + notExists_ = notExists; + } + + @Override + public Expr negate() { + return new ExistsPredicate((Subquery)getChild(0), !notExists_); + } + + /** + * Copy c'tor used in clone. + */ + public ExistsPredicate(ExistsPredicate other) { + super(other); + notExists_ = other.notExists_; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (isAnalyzed_) return; + super.analyze(analyzer); + } + + @Override + protected void toThrift(TExprNode msg) { + // Cannot serialize a nested predicate + Preconditions.checkState(false); + } + + @Override + public Expr clone() { return new ExistsPredicate(this); } + + @Override + public String toSqlImpl() { + StringBuilder strBuilder = new StringBuilder(); + if (notExists_) strBuilder.append("NOT "); + strBuilder.append("EXISTS "); + strBuilder.append(getChild(0).toSql()); + return strBuilder.toString(); + } +}
