http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/InsertStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/InsertStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/InsertStmt.java deleted file mode 100644 index c5965db..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/InsertStmt.java +++ /dev/null @@ -1,697 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -import com.cloudera.impala.planner.TableSink; -import com.google.common.collect.ImmutableList; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.cloudera.impala.authorization.Privilege; -import com.cloudera.impala.authorization.PrivilegeRequestBuilder; -import com.cloudera.impala.catalog.Column; -import com.cloudera.impala.catalog.HBaseTable; -import com.cloudera.impala.catalog.HdfsTable; -import com.cloudera.impala.catalog.KuduTable; -import com.cloudera.impala.catalog.Table; -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.catalog.View; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.FileSystemUtil; -import com.cloudera.impala.planner.DataSink; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -/** - * Representation of a single insert statement, including the select statement - * whose results are to be inserted. - */ -public class InsertStmt extends StatementBase { - private final static Logger LOG = LoggerFactory.getLogger(InsertStmt.class); - - // Target table name as seen by the parser - private final TableName originalTableName_; - - // Differentiates between INSERT INTO and INSERT OVERWRITE. - private final boolean overwrite_; - - // List of column:value elements from the PARTITION (...) clause. - // Set to null if no partition was given. - private final List<PartitionKeyValue> partitionKeyValues_; - - // User-supplied hints to control hash partitioning before the table sink in the plan. - private final List<String> planHints_; - - // False if the original insert statement had a query statement, true if we need to - // auto-generate one (for insert into tbl()) during analysis. - private final boolean needsGeneratedQueryStatement_; - - // The column permutation is specified by writing INSERT INTO tbl(col3, col1, col2...) - // - // It is a mapping from select-list expr index to (non-partition) output column. If - // null, will be set to the default permutation of all non-partition columns in Hive - // order. - // - // A column is said to be 'mentioned' if it occurs either in the column permutation, or - // the PARTITION clause. If columnPermutation is null, all non-partition columns are - // considered mentioned. - // - // Between them, the columnPermutation and the set of partitionKeyValues must mention to - // every partition column in the target table exactly once. Other columns, if not - // explicitly mentioned, will be assigned NULL values. Partition columns are not - // defaulted to NULL by design, and are not just for NULL-valued partition slots. - // - // Dynamic partition keys may occur in either the permutation or the PARTITION - // clause. Partition columns with static values may only be mentioned in the PARTITION - // clause, where the static value is specified. - private final List<String> columnPermutation_; - - ///////////////////////////////////////// - // BEGIN: Members that need to be reset() - - // List of inline views that may be referenced in queryStmt. - private final WithClause withClause_; - - // Target table into which to insert. May be qualified by analyze() - private TableName targetTableName_; - - // Select or union whose results are to be inserted. If null, will be set after - // analysis. - private QueryStmt queryStmt_; - - // Set in analyze(). Contains metadata of target table to determine type of sink. - private Table table_; - - // Set in analyze(). Exprs corresponding to the partitionKeyValues, - private List<Expr> partitionKeyExprs_ = Lists.newArrayList(); - - // Indicates whether this insert stmt has a shuffle or noshuffle plan hint. - // Both flags may be false. Only one of them may be true, not both. - // Shuffle forces data repartitioning before then data sink, and noshuffle - // prevents it. Set in analyze() based on planHints_. - private boolean hasShuffleHint_ = false; - private boolean hasNoShuffleHint_ = false; - - // Output expressions that produce the final results to write to the target table. May - // include casts, and NullLiterals where an output column isn't explicitly mentioned. - // Set in prepareExpressions(). The i'th expr produces the i'th column of the target - // table. - private ArrayList<Expr> resultExprs_ = Lists.newArrayList(); - - // END: Members that need to be reset() - ///////////////////////////////////////// - - // For tables with primary keys, indicates if duplicate key errors are ignored. - private final boolean ignoreDuplicates_; - - public InsertStmt(WithClause withClause, TableName targetTable, boolean overwrite, - List<PartitionKeyValue> partitionKeyValues, List<String> planHints, - QueryStmt queryStmt, List<String> columnPermutation, boolean ignoreDuplicates) { - withClause_ = withClause; - targetTableName_ = targetTable; - originalTableName_ = targetTableName_; - overwrite_ = overwrite; - partitionKeyValues_ = partitionKeyValues; - planHints_ = planHints; - queryStmt_ = queryStmt; - needsGeneratedQueryStatement_ = (queryStmt == null); - columnPermutation_ = columnPermutation; - table_ = null; - ignoreDuplicates_ = ignoreDuplicates; - } - - /** - * C'tor used in clone(). - */ - private InsertStmt(InsertStmt other) { - super(other); - withClause_ = other.withClause_ != null ? other.withClause_.clone() : null; - targetTableName_ = other.targetTableName_; - originalTableName_ = other.originalTableName_; - overwrite_ = other.overwrite_; - partitionKeyValues_ = other.partitionKeyValues_; - planHints_ = other.planHints_; - queryStmt_ = other.queryStmt_ != null ? other.queryStmt_.clone() : null; - needsGeneratedQueryStatement_ = other.needsGeneratedQueryStatement_; - columnPermutation_ = other.columnPermutation_; - table_ = other.table_; - ignoreDuplicates_ = other.ignoreDuplicates_; - } - - @Override - public void reset() { - super.reset(); - if (withClause_ != null) withClause_.reset(); - targetTableName_ = originalTableName_; - queryStmt_.reset(); - table_ = null; - partitionKeyExprs_.clear(); - hasShuffleHint_ = false; - hasNoShuffleHint_ = false; - resultExprs_.clear(); - } - - @Override - public InsertStmt clone() { return new InsertStmt(this); } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed()) return; - super.analyze(analyzer); - try { - if (withClause_ != null) withClause_.analyze(analyzer); - } catch (AnalysisException e) { - // Ignore AnalysisExceptions if tables are missing to ensure the maximum number - // of missing tables can be collected before failing analyze(). - if (analyzer.getMissingTbls().isEmpty()) throw e; - } - - List<Expr> selectListExprs = null; - if (!needsGeneratedQueryStatement_) { - try { - // Use a child analyzer for the query stmt to properly scope WITH-clause - // views and to ignore irrelevant ORDER BYs. - Analyzer queryStmtAnalyzer = new Analyzer(analyzer); - queryStmt_.analyze(queryStmtAnalyzer); - // Subqueries need to be rewritten by the StmtRewriter first. - if (analyzer.containsSubquery()) return; - // Use getResultExprs() and not getBaseTblResultExprs() here because the final - // substitution with TupleIsNullPredicate() wrapping happens in planning. - selectListExprs = Expr.cloneList(queryStmt_.getResultExprs()); - } catch (AnalysisException e) { - if (analyzer.getMissingTbls().isEmpty()) throw e; - } - } else { - selectListExprs = Lists.newArrayList(); - } - - // Set target table and perform table-type specific analysis and auth checking. - // Also checks if the target table is missing. - setTargetTable(analyzer); - - // Abort analysis if there are any missing tables beyond this point. - if (!analyzer.getMissingTbls().isEmpty()) { - throw new AnalysisException("Found missing tables. Aborting analysis."); - } - - boolean isHBaseTable = (table_ instanceof HBaseTable); - int numClusteringCols = isHBaseTable ? 0 : table_.getNumClusteringCols(); - - // Analysis of the INSERT statement from this point is basically the act of matching - // the set of output columns (which come from a column permutation, perhaps - // implicitly, and the PARTITION clause) to the set of input columns (which come from - // the select-list and any statically-valued columns in the PARTITION clause). - // - // First, we compute the set of mentioned columns, and reject statements that refer to - // non-existent columns, or duplicates (we must check both the column permutation, and - // the set of partition keys). Next, we check that all partition columns are - // mentioned. During this process we build the map from select-list expr index to - // column in the targeted table. - // - // Then we check that the select-list contains exactly the right number of expressions - // for all mentioned columns which are not statically-valued partition columns (which - // get their expressions from partitionKeyValues). - // - // Finally, prepareExpressions analyzes the expressions themselves, and confirms that - // they are type-compatible with the target columns. Where columns are not mentioned - // (and by this point, we know that missing columns are not partition columns), - // prepareExpressions assigns them a NULL literal expressions. - - // An null permutation clause is the same as listing all non-partition columns in - // order. - List<String> analysisColumnPermutation = columnPermutation_; - if (analysisColumnPermutation == null) { - analysisColumnPermutation = Lists.newArrayList(); - ArrayList<Column> tableColumns = table_.getColumns(); - for (int i = numClusteringCols; i < tableColumns.size(); ++i) { - analysisColumnPermutation.add(tableColumns.get(i).getName()); - } - } - - // selectExprTargetColumns maps from select expression index to a column in the target - // table. It will eventually include all mentioned columns that aren't static-valued - // partition columns. - ArrayList<Column> selectExprTargetColumns = Lists.newArrayList(); - - // Tracks the name of all columns encountered in either the permutation clause or the - // partition clause to detect duplicates. - Set<String> mentionedColumnNames = Sets.newHashSet(); - for (String columnName: analysisColumnPermutation) { - Column column = table_.getColumn(columnName); - if (column == null) { - throw new AnalysisException( - "Unknown column '" + columnName + "' in column permutation"); - } - - if (!mentionedColumnNames.add(columnName)) { - throw new AnalysisException( - "Duplicate column '" + columnName + "' in column permutation"); - } - selectExprTargetColumns.add(column); - } - - int numStaticPartitionExprs = 0; - if (partitionKeyValues_ != null) { - for (PartitionKeyValue pkv: partitionKeyValues_) { - Column column = table_.getColumn(pkv.getColName()); - if (column == null) { - throw new AnalysisException("Unknown column '" + pkv.getColName() + - "' in partition clause"); - } - - if (column.getPosition() >= numClusteringCols) { - throw new AnalysisException( - "Column '" + pkv.getColName() + "' is not a partition column"); - } - - if (!mentionedColumnNames.add(pkv.getColName())) { - throw new AnalysisException( - "Duplicate column '" + pkv.getColName() + "' in partition clause"); - } - if (!pkv.isDynamic()) { - numStaticPartitionExprs++; - } else { - selectExprTargetColumns.add(column); - } - } - } - - // Checks that exactly all columns in the target table are assigned an expr. - checkColumnCoverage(selectExprTargetColumns, mentionedColumnNames, - selectListExprs.size(), numStaticPartitionExprs); - - // Make sure static partition key values only contain const exprs. - if (partitionKeyValues_ != null) { - for (PartitionKeyValue kv: partitionKeyValues_) { - kv.analyze(analyzer); - } - } - - // Populate partitionKeyExprs from partitionKeyValues and selectExprTargetColumns - prepareExpressions(selectExprTargetColumns, selectListExprs, table_, analyzer); - // Analyze plan hints at the end to prefer reporting other error messages first - // (e.g., the PARTITION clause is not applicable to unpartitioned and HBase tables). - analyzePlanHints(analyzer); - } - - /** - * Sets table_ based on targetTableName_ and performs table-type specific analysis: - * - Partition clause is invalid for unpartitioned Hdfs tables and HBase tables - * - Overwrite is invalid for HBase tables - * - Check INSERT privileges as well as write access to Hdfs paths - * - Cannot insert into a view - * Adds table_ to the analyzer's descriptor table if analysis succeeds. - */ - private void setTargetTable(Analyzer analyzer) throws AnalysisException { - // If the table has not yet been set, load it from the Catalog. This allows for - // callers to set a table to analyze that may not actually be created in the Catalog. - // One example use case is CREATE TABLE AS SELECT which must run analysis on the - // INSERT before the table has actually been created. - if (table_ == null) { - if (!targetTableName_.isFullyQualified()) { - targetTableName_ = - new TableName(analyzer.getDefaultDb(), targetTableName_.getTbl()); - } - table_ = analyzer.getTable(targetTableName_, Privilege.INSERT); - } else { - targetTableName_ = new TableName(table_.getDb().getName(), table_.getName()); - PrivilegeRequestBuilder pb = new PrivilegeRequestBuilder(); - analyzer.registerPrivReq(pb.onTable(table_.getDb().getName(), table_.getName()) - .allOf(Privilege.INSERT).toRequest()); - } - - // We do not support inserting into views. - if (table_ instanceof View) { - throw new AnalysisException( - String.format("Impala does not support inserting into views: %s", - table_.getFullName())); - } - - for (Column c: table_.getColumns()) { - if (!c.getType().isSupported()) { - throw new AnalysisException(String.format("Unable to INSERT into target table " + - "(%s) because the column '%s' has an unsupported type '%s'.", - targetTableName_, c.getName(), c.getType().toSql())); - } - } - - boolean isHBaseTable = (table_ instanceof HBaseTable); - int numClusteringCols = isHBaseTable ? 0 : table_.getNumClusteringCols(); - - if (partitionKeyValues_ != null && numClusteringCols == 0) { - if (isHBaseTable) { - throw new AnalysisException("PARTITION clause is not valid for INSERT into " + - "HBase tables. '" + targetTableName_ + "' is an HBase table"); - - } else { - // Unpartitioned table, but INSERT has PARTITION clause - throw new AnalysisException("PARTITION clause is only valid for INSERT into " + - "partitioned table. '" + targetTableName_ + "' is not partitioned"); - } - } - - if (table_ instanceof HdfsTable) { - HdfsTable hdfsTable = (HdfsTable) table_; - if (!hdfsTable.hasWriteAccess()) { - throw new AnalysisException(String.format("Unable to INSERT into target table " + - "(%s) because Impala does not have WRITE access to at least one HDFS path" + - ": %s", targetTableName_, hdfsTable.getFirstLocationWithoutWriteAccess())); - } - StringBuilder error = new StringBuilder(); - hdfsTable.parseSkipHeaderLineCount(error); - if (error.length() > 0) throw new AnalysisException(error.toString()); - try { - if (!FileSystemUtil.isImpalaWritableFilesystem(hdfsTable.getLocation())) { - throw new AnalysisException(String.format("Unable to INSERT into target " + - "table (%s) because %s is not a supported filesystem.", targetTableName_, - hdfsTable.getLocation())); - } - } catch (IOException e) { - throw new AnalysisException(String.format("Unable to INSERT into target " + - "table (%s): %s.", targetTableName_, e.getMessage()), e); - } - for (int colIdx = 0; colIdx < numClusteringCols; ++colIdx) { - Column col = hdfsTable.getColumns().get(colIdx); - // Hive has a number of issues handling BOOLEAN partition columns (see HIVE-6590). - // Instead of working around the Hive bugs, INSERT is disabled for BOOLEAN - // partitions in Impala. Once the Hive JIRA is resolved, we can remove this - // analysis check. - if (col.getType() == Type.BOOLEAN) { - throw new AnalysisException(String.format("INSERT into table with BOOLEAN " + - "partition column (%s) is not supported: %s", col.getName(), - targetTableName_)); - } - } - } - - if (table_ instanceof KuduTable) { - if (overwrite_) { - throw new AnalysisException("INSERT OVERWRITE not supported for Kudu tables."); - } - if (partitionKeyValues_ != null && !partitionKeyValues_.isEmpty()) { - throw new AnalysisException( - "Partition specifications are not supported for Kudu tables."); - } - } - - if (isHBaseTable && overwrite_) { - throw new AnalysisException("HBase doesn't have a way to perform INSERT OVERWRITE"); - } - - // Add target table to descriptor table. - analyzer.getDescTbl().addReferencedTable(table_); - } - - /** - * Checks that the column permutation + select list + static partition exprs + - * dynamic partition exprs collectively cover exactly all columns in the target table - * (not more of fewer). - */ - private void checkColumnCoverage(ArrayList<Column> selectExprTargetColumns, - Set<String> mentionedColumnNames, int numSelectListExprs, - int numStaticPartitionExprs) throws AnalysisException { - boolean isHBaseTable = (table_ instanceof HBaseTable); - int numClusteringCols = isHBaseTable ? 0 : table_.getNumClusteringCols(); - // Check that all columns are mentioned by the permutation and partition clauses - if (selectExprTargetColumns.size() + numStaticPartitionExprs != - table_.getColumns().size()) { - // We've already ruled out too many columns in the permutation and partition clauses - // by checking that there are no duplicates and that every column mentioned actually - // exists. So all columns aren't mentioned in the query. If the unmentioned columns - // include partition columns, this is an error. - List<String> missingColumnNames = Lists.newArrayList(); - for (Column column: table_.getColumns()) { - if (!mentionedColumnNames.contains(column.getName())) { - // HBase tables have a single row-key column which is always in position 0. It - // must be mentioned, since it is invalid to set it to NULL (which would - // otherwise happen by default). - if (isHBaseTable && column.getPosition() == 0) { - throw new AnalysisException("Row-key column '" + column.getName() + - "' must be explicitly mentioned in column permutation."); - } - if (column.getPosition() < numClusteringCols) { - missingColumnNames.add(column.getName()); - } - } - } - - if (!missingColumnNames.isEmpty()) { - throw new AnalysisException( - "Not enough partition columns mentioned in query. Missing columns are: " + - Joiner.on(", ").join(missingColumnNames)); - } - } - - // Expect the selectListExpr to have entries for every target column - if (selectExprTargetColumns.size() != numSelectListExprs) { - String comparator = - (selectExprTargetColumns.size() < numSelectListExprs) ? "fewer" : "more"; - String partitionClause = - (partitionKeyValues_ == null) ? "returns" : "and PARTITION clause return"; - - // If there was no column permutation provided, the error is that the select-list - // has the wrong number of expressions compared to the number of columns in the - // table. If there was a column permutation, then the mismatch is between the - // select-list and the permutation itself. - if (columnPermutation_ == null) { - int totalColumnsMentioned = numSelectListExprs + numStaticPartitionExprs; - throw new AnalysisException(String.format( - "Target table '%s' has %s columns (%s) than the SELECT / VALUES clause %s" + - " (%s)", table_.getFullName(), comparator, - table_.getColumns().size(), partitionClause, totalColumnsMentioned)); - } else { - String partitionPrefix = - (partitionKeyValues_ == null) ? "mentions" : "and PARTITION clause mention"; - throw new AnalysisException(String.format( - "Column permutation %s %s columns (%s) than " + - "the SELECT / VALUES clause %s (%s)", partitionPrefix, comparator, - selectExprTargetColumns.size(), partitionClause, numSelectListExprs)); - } - } - } - - /** - * Performs three final parts of the analysis: - * 1. Checks type compatibility between all expressions and their targets - * - * 2. Populates partitionKeyExprs with type-compatible expressions, in Hive - * partition-column order, for all partition columns - * - * 3. Populates resultExprs_ with type-compatible expressions, in Hive column order, - * for all expressions in the select-list. Unmentioned columns are assigned NULL literal - * expressions. - * - * If necessary, adds casts to the expressions to make them compatible with the type of - * the corresponding column. - * - * @throws AnalysisException - * If an expression is not compatible with its target column - */ - private void prepareExpressions(List<Column> selectExprTargetColumns, - List<Expr> selectListExprs, Table tbl, Analyzer analyzer) - throws AnalysisException { - // Temporary lists of partition key exprs and names in an arbitrary order. - List<Expr> tmpPartitionKeyExprs = new ArrayList<Expr>(); - List<String> tmpPartitionKeyNames = new ArrayList<String>(); - - int numClusteringCols = (tbl instanceof HBaseTable) ? 0 : tbl.getNumClusteringCols(); - - // Check dynamic partition columns for type compatibility. - for (int i = 0; i < selectListExprs.size(); ++i) { - Column targetColumn = selectExprTargetColumns.get(i); - Expr compatibleExpr = checkTypeCompatibility( - targetTableName_.toString(), targetColumn, selectListExprs.get(i)); - if (targetColumn.getPosition() < numClusteringCols) { - // This is a dynamic clustering column - tmpPartitionKeyExprs.add(compatibleExpr); - tmpPartitionKeyNames.add(targetColumn.getName()); - } - selectListExprs.set(i, compatibleExpr); - } - - // Check static partition columns, dynamic entries in partitionKeyValues will already - // be in selectExprTargetColumns and therefore are ignored in this loop - if (partitionKeyValues_ != null) { - for (PartitionKeyValue pkv: partitionKeyValues_) { - if (pkv.isStatic()) { - // tableColumns is guaranteed to exist after the earlier analysis checks - Column tableColumn = table_.getColumn(pkv.getColName()); - Expr compatibleExpr = checkTypeCompatibility( - targetTableName_.toString(), tableColumn, pkv.getValue()); - tmpPartitionKeyExprs.add(compatibleExpr); - tmpPartitionKeyNames.add(pkv.getColName()); - } - } - } - - // Reorder the partition key exprs and names to be consistent with the target table - // declaration. We need those exprs in the original order to create the corresponding - // Hdfs folder structure correctly. - for (Column c: table_.getColumns()) { - for (int j = 0; j < tmpPartitionKeyNames.size(); ++j) { - if (c.getName().equals(tmpPartitionKeyNames.get(j))) { - partitionKeyExprs_.add(tmpPartitionKeyExprs.get(j)); - break; - } - } - } - - Preconditions.checkState(partitionKeyExprs_.size() == numClusteringCols); - // Make sure we have stats for partitionKeyExprs - for (Expr expr: partitionKeyExprs_) { - expr.analyze(analyzer); - } - - // Finally, 'undo' the permutation so that the selectListExprs are in Hive column - // order, and add NULL expressions to all missing columns. - for (Column tblColumn: table_.getColumnsInHiveOrder()) { - boolean matchFound = false; - for (int i = 0; i < selectListExprs.size(); ++i) { - if (selectExprTargetColumns.get(i).getName().equals(tblColumn.getName())) { - resultExprs_.add(selectListExprs.get(i)); - matchFound = true; - break; - } - } - // If no match is found, either the column is a clustering column with a static - // value, or it was unmentioned and therefore should have a NULL select-list - // expression. - if (!matchFound) { - if (tblColumn.getPosition() >= numClusteringCols) { - // Unmentioned non-clustering columns get NULL literals with the appropriate - // target type because Parquet cannot handle NULL_TYPE (IMPALA-617). - resultExprs_.add(NullLiteral.create(tblColumn.getType())); - } - } - } - // TODO: Check that HBase row-key columns are not NULL? See IMPALA-406 - if (needsGeneratedQueryStatement_) { - // Build a query statement that returns NULL for every column - List<SelectListItem> selectListItems = Lists.newArrayList(); - for(Expr e: resultExprs_) { - selectListItems.add(new SelectListItem(e, null)); - } - SelectList selectList = new SelectList(selectListItems); - queryStmt_ = new SelectStmt(selectList, null, null, null, null, null, null); - queryStmt_.analyze(analyzer); - } - } - - private void analyzePlanHints(Analyzer analyzer) throws AnalysisException { - if (planHints_ == null) return; - if (!planHints_.isEmpty() && table_ instanceof HBaseTable) { - throw new AnalysisException("INSERT hints are only supported for inserting into " + - "Hdfs tables."); - } - for (String hint: planHints_) { - if (hint.equalsIgnoreCase("SHUFFLE")) { - if (hasNoShuffleHint_) { - throw new AnalysisException("Conflicting INSERT hint: " + hint); - } - hasShuffleHint_ = true; - analyzer.setHasPlanHints(); - } else if (hint.equalsIgnoreCase("NOSHUFFLE")) { - if (hasShuffleHint_) { - throw new AnalysisException("Conflicting INSERT hint: " + hint); - } - hasNoShuffleHint_ = true; - analyzer.setHasPlanHints(); - } else { - analyzer.addWarning("INSERT hint not recognized: " + hint); - } - } - // Both flags may be false or one of them may be true, but not both. - Preconditions.checkState((!hasShuffleHint_ && !hasNoShuffleHint_) - || (hasShuffleHint_ ^ hasNoShuffleHint_)); - } - - public List<String> getPlanHints() { return planHints_; } - public TableName getTargetTableName() { return targetTableName_; } - public Table getTargetTable() { return table_; } - public void setTargetTable(Table table) { this.table_ = table; } - public boolean isOverwrite() { return overwrite_; } - - /** - * Only valid after analysis - */ - public QueryStmt getQueryStmt() { return queryStmt_; } - public void setQueryStmt(QueryStmt stmt) { queryStmt_ = stmt; } - public List<Expr> getPartitionKeyExprs() { return partitionKeyExprs_; } - public boolean hasShuffleHint() { return hasShuffleHint_; } - public boolean hasNoShuffleHint() { return hasNoShuffleHint_; } - public ArrayList<Expr> getResultExprs() { return resultExprs_; } - - public DataSink createDataSink() { - // analyze() must have been called before. - Preconditions.checkState(table_ != null); - return TableSink.create(table_, TableSink.Op.INSERT, partitionKeyExprs_, - ImmutableList.<Integer>of(), overwrite_, ignoreDuplicates_); - } - - /** - * Substitutes the result expressions and the partition key expressions with smap. - * Preserves the original types of those expressions during the substitution. - */ - public void substituteResultExprs(ExprSubstitutionMap smap, Analyzer analyzer) { - resultExprs_ = Expr.substituteList(resultExprs_, smap, analyzer, true); - partitionKeyExprs_ = Expr.substituteList(partitionKeyExprs_, smap, analyzer, true); - } - - @Override - public String toSql() { - StringBuilder strBuilder = new StringBuilder(); - - if (withClause_ != null) strBuilder.append(withClause_.toSql() + " "); - - strBuilder.append("INSERT "); - if (overwrite_) { - strBuilder.append("OVERWRITE "); - } else { - if (ignoreDuplicates_) strBuilder.append("IGNORE "); - strBuilder.append("INTO "); - } - strBuilder.append("TABLE " + originalTableName_); - if (columnPermutation_ != null) { - strBuilder.append("("); - strBuilder.append(Joiner.on(", ").join(columnPermutation_)); - strBuilder.append(")"); - } - if (partitionKeyValues_ != null) { - List<String> values = Lists.newArrayList(); - for (PartitionKeyValue pkv: partitionKeyValues_) { - values.add(pkv.getColName() + - (pkv.getValue() != null ? ("=" + pkv.getValue().toSql()) : "")); - } - strBuilder.append(" PARTITION (" + Joiner.on(", ").join(values) + ")"); - } - if (planHints_ != null) { - strBuilder.append(" " + ToSqlUtils.getPlanHintsSql(planHints_)); - } - if (!needsGeneratedQueryStatement_) { - strBuilder.append(" " + queryStmt_.toSql()); - } - return strBuilder.toString(); - } -}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/IsNotEmptyPredicate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/IsNotEmptyPredicate.java b/fe/src/main/java/com/cloudera/impala/analysis/IsNotEmptyPredicate.java deleted file mode 100644 index fd07a32..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/IsNotEmptyPredicate.java +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TExprNodeType; -import com.google.common.base.Preconditions; - -/** - * Predicate that checks whether a collection is empty or not. - * This predicate is not user-accessible from SQL, and may be - * generated as a performance optimization for certain queries. - * TODO: Pass this Predicate as a TExprNodeType.FUNCTION_CALL - * to the BE just like the rest of our Predicates. This is not yet - * done to avoid invasive changes required in FE/BE to deal with - * resolution of functions with complex-types arguments, - */ -public class IsNotEmptyPredicate extends Predicate { - - public IsNotEmptyPredicate(Expr collectionExpr) { - super(); - Preconditions.checkNotNull(collectionExpr); - children_.add(collectionExpr); - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - super.analyze(analyzer); - if (!getChild(0).getType().isCollectionType()) { - throw new AnalysisException("Operand must be a collection type: " - + getChild(0).toSql() + " is of type " + getChild(0).getType()); - } - // Avoid influencing cardinality estimates. - selectivity_ = 1.0; - if (getChild(0).hasCost()) evalCost_ = getChild(0).getCost() + IS_NOT_EMPTY_COST; - } - - @Override - public String toSqlImpl() { return "!empty(" + getChild(0).toSql() + ")"; } - - @Override - protected void toThrift(TExprNode msg) { - msg.node_type = TExprNodeType.IS_NOT_EMPTY_PRED; - } - - @Override - public Expr clone() { return new IsNotEmptyPredicate(getChild(0).clone()); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/IsNullPredicate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/IsNullPredicate.java b/fe/src/main/java/com/cloudera/impala/analysis/IsNullPredicate.java deleted file mode 100644 index 6a75b5f..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/IsNullPredicate.java +++ /dev/null @@ -1,186 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.catalog.Db; -import com.cloudera.impala.catalog.Function; -import com.cloudera.impala.catalog.Function.CompareMode; -import com.cloudera.impala.catalog.ScalarFunction; -import com.cloudera.impala.catalog.ScalarType; -import com.cloudera.impala.catalog.Table; -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.Reference; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TExprNodeType; -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -public class IsNullPredicate extends Predicate { - private final boolean isNotNull_; - - private static final String IS_NULL = "is_null_pred"; - private static final String IS_NOT_NULL = "is_not_null_pred"; - - public IsNullPredicate(Expr e, boolean isNotNull) { - super(); - this.isNotNull_ = isNotNull; - Preconditions.checkNotNull(e); - children_.add(e); - } - - /** - * Copy c'tor used in clone(). - */ - protected IsNullPredicate(IsNullPredicate other) { - super(other); - isNotNull_ = other.isNotNull_; - } - - public boolean isNotNull() { return isNotNull_; } - - public static void initBuiltins(Db db) { - for (Type t: Type.getSupportedTypes()) { - if (t.isNull()) continue; - String isNullSymbol; - if (t.isBoolean()) { - isNullSymbol = "_ZN6impala15IsNullPredicate6IsNullIN10impala_udf10BooleanValE" + - "EES3_PNS2_15FunctionContextERKT_"; - } else { - String udfType = Function.getUdfType(t); - isNullSymbol = "_ZN6impala15IsNullPredicate6IsNullIN10impala_udf" + - udfType.length() + udfType + - "EEENS2_10BooleanValEPNS2_15FunctionContextERKT_"; - } - db.addBuiltin(ScalarFunction.createBuiltinOperator( - IS_NULL, isNullSymbol, Lists.newArrayList(t), Type.BOOLEAN)); - - String isNotNullSymbol = isNullSymbol.replace("6IsNull", "9IsNotNull"); - db.addBuiltin(ScalarFunction.createBuiltinOperator( - IS_NOT_NULL, isNotNullSymbol, Lists.newArrayList(t), Type.BOOLEAN)); - } - } - - @Override - public boolean equals(Object obj) { - if (!super.equals(obj)) return false; - return ((IsNullPredicate) obj).isNotNull_ == isNotNull_; - } - - @Override - public String toSqlImpl() { - return getChild(0).toSql() + (isNotNull_ ? " IS NOT NULL" : " IS NULL"); - } - - @Override - public String debugString() { - return Objects.toStringHelper(this) - .add("notNull", isNotNull_) - .addValue(super.debugString()) - .toString(); - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - super.analyze(analyzer); - - if (contains(Subquery.class)) { - if (getChild(0) instanceof ExistsPredicate) { - // Replace the EXISTS subquery with a BoolLiteral as it can never return - // a null value. - setChild(0, new BoolLiteral(true)); - getChild(0).analyze(analyzer); - } else if (!getChild(0).contains(Expr.IS_SCALAR_SUBQUERY)) { - // We only support scalar subqueries in an IS NULL predicate because - // they can be rewritten into a join. - // TODO: Add support for InPredicates and BinaryPredicates with - // subqueries when we implement independent subquery evaluation. - // TODO: Handle arbitrary UDA/Udfs - throw new AnalysisException("Unsupported IS NULL predicate that contains " + - "a subquery: " + toSqlImpl()); - } - } - - // Make sure the BE never sees TYPE_NULL - if (getChild(0).getType().isNull()) { - uncheckedCastChild(ScalarType.BOOLEAN, 0); - } - - if (getChild(0).getType().isComplexType()) { - String errorMsg = (isNotNull_ ? "IS NOT NULL" : "IS NULL") + - " predicate does not support complex types: "; - throw new AnalysisException(errorMsg + toSqlImpl()); - } - - if (isNotNull_) { - fn_ = getBuiltinFunction( - analyzer, IS_NOT_NULL, collectChildReturnTypes(), CompareMode.IS_IDENTICAL); - } else { - fn_ = getBuiltinFunction( - analyzer, IS_NULL, collectChildReturnTypes(), CompareMode.IS_IDENTICAL); - } - if (getChild(0).hasCost()) evalCost_ = getChild(0).getCost() + IS_NULL_COST; - - // determine selectivity - // TODO: increase this to make sure we don't end up favoring broadcast joins - // due to underestimated cardinalities? - Reference<SlotRef> slotRefRef = new Reference<SlotRef>(); - if (isSingleColumnPredicate(slotRefRef, null)) { - SlotDescriptor slotDesc = slotRefRef.getRef().getDesc(); - if (!slotDesc.getStats().hasNulls()) return; - Table table = slotDesc.getParent().getTable(); - if (table != null && table.getNumRows() > 0) { - long numRows = table.getNumRows(); - if (isNotNull_) { - selectivity_ = - (double) (numRows - slotDesc.getStats().getNumNulls()) / (double) numRows; - } else { - selectivity_ = (double) slotDesc.getStats().getNumNulls() / (double) numRows; - } - selectivity_ = Math.max(0.0, Math.min(1.0, selectivity_)); - } - } - } - - @Override - protected void toThrift(TExprNode msg) { - msg.node_type = TExprNodeType.FUNCTION_CALL; - } - - /* - * If predicate is of the form "<SlotRef> IS [NOT] NULL", returns the - * SlotRef. - */ - @Override - public SlotRef getBoundSlot() { - return getChild(0).unwrapSlotRef(true); - } - - /** - * Negates an IsNullPredicate. - */ - @Override - public Expr negate() { - return new IsNullPredicate(getChild(0), !isNotNull_); - } - - @Override - public Expr clone() { return new IsNullPredicate(this); } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/JoinOperator.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/JoinOperator.java b/fe/src/main/java/com/cloudera/impala/analysis/JoinOperator.java deleted file mode 100644 index d50861b..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/JoinOperator.java +++ /dev/null @@ -1,112 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.thrift.TJoinOp; - -public enum JoinOperator { - INNER_JOIN("INNER JOIN", TJoinOp.INNER_JOIN), - LEFT_OUTER_JOIN("LEFT OUTER JOIN", TJoinOp.LEFT_OUTER_JOIN), - LEFT_SEMI_JOIN("LEFT SEMI JOIN", TJoinOp.LEFT_SEMI_JOIN), - LEFT_ANTI_JOIN("LEFT ANTI JOIN", TJoinOp.LEFT_ANTI_JOIN), - RIGHT_OUTER_JOIN("RIGHT OUTER JOIN", TJoinOp.RIGHT_OUTER_JOIN), - RIGHT_SEMI_JOIN("RIGHT SEMI JOIN", TJoinOp.RIGHT_SEMI_JOIN), - RIGHT_ANTI_JOIN("RIGHT ANTI JOIN", TJoinOp.RIGHT_ANTI_JOIN), - FULL_OUTER_JOIN("FULL OUTER JOIN", TJoinOp.FULL_OUTER_JOIN), - CROSS_JOIN("CROSS JOIN", TJoinOp.CROSS_JOIN), - // Variant of the LEFT ANTI JOIN that is used for the rewrite of - // NOT IN subqueries. It can have a single equality join conjunct - // that returns TRUE when the rhs is NULL. - NULL_AWARE_LEFT_ANTI_JOIN("NULL AWARE LEFT ANTI JOIN", - TJoinOp.NULL_AWARE_LEFT_ANTI_JOIN); - - private final String description_; - private final TJoinOp thriftJoinOp_; - - private JoinOperator(String description, TJoinOp thriftJoinOp) { - this.description_ = description; - this.thriftJoinOp_ = thriftJoinOp; - } - - @Override - public String toString() { - return description_; - } - - public TJoinOp toThrift() { - return thriftJoinOp_; - } - - public boolean isInnerJoin() { - return this == INNER_JOIN; - } - - public boolean isLeftOuterJoin() { return this == LEFT_OUTER_JOIN; } - public boolean isRightOuterJoin() { return this == RIGHT_OUTER_JOIN; } - - public boolean isOuterJoin() { - return this == LEFT_OUTER_JOIN - || this == RIGHT_OUTER_JOIN - || this == FULL_OUTER_JOIN; - } - - public boolean isSemiJoin() { - return this == JoinOperator.LEFT_SEMI_JOIN || this == JoinOperator.LEFT_ANTI_JOIN || - this == JoinOperator.RIGHT_SEMI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN || - this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN; - } - - public boolean isLeftSemiJoin() { - return this == JoinOperator.LEFT_SEMI_JOIN || this == JoinOperator.LEFT_ANTI_JOIN || - this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN; - } - - public boolean isRightSemiJoin() { - return this == JoinOperator.RIGHT_SEMI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN; - } - - public boolean isCrossJoin() { - return this == JoinOperator.CROSS_JOIN; - } - - public boolean isFullOuterJoin() { - return this == JoinOperator.FULL_OUTER_JOIN; - } - - public boolean isNullAwareLeftAntiJoin() { - return this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN; - } - - public boolean isAntiJoin() { - return this == JoinOperator.LEFT_ANTI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN || - this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN; - } - - public JoinOperator invert() { - switch (this) { - case LEFT_OUTER_JOIN: return RIGHT_OUTER_JOIN; - case RIGHT_OUTER_JOIN: return LEFT_OUTER_JOIN; - case LEFT_SEMI_JOIN: return RIGHT_SEMI_JOIN; - case RIGHT_SEMI_JOIN: return LEFT_SEMI_JOIN; - case LEFT_ANTI_JOIN: return RIGHT_ANTI_JOIN; - case RIGHT_ANTI_JOIN: return LEFT_ANTI_JOIN; - case NULL_AWARE_LEFT_ANTI_JOIN: throw new IllegalStateException("Not implemented"); - default: return this; - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/LikePredicate.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/LikePredicate.java b/fe/src/main/java/com/cloudera/impala/analysis/LikePredicate.java deleted file mode 100644 index cb006bf..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/LikePredicate.java +++ /dev/null @@ -1,169 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; - -import com.cloudera.impala.catalog.Db; -import com.cloudera.impala.catalog.Function.CompareMode; -import com.cloudera.impala.catalog.ScalarFunction; -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TExprNodeType; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -public class LikePredicate extends Predicate { - enum Operator { - LIKE("LIKE"), - ILIKE("ILIKE"), - RLIKE("RLIKE"), - REGEXP("REGEXP"), - IREGEXP("IREGEXP"); - - private final String description_; - - private Operator(String description) { - this.description_ = description; - } - - @Override - public String toString() { - return description_; - } - } - - public static void initBuiltins(Db db) { - db.addBuiltin(ScalarFunction.createBuiltin( - Operator.LIKE.name(), Lists.<Type>newArrayList(Type.STRING, Type.STRING), - false, Type.BOOLEAN, "_ZN6impala13LikePredicate4LikeEPN10impala_udf15FunctionContextERKNS1_9StringValES6_", - "_ZN6impala13LikePredicate11LikePrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE", - "_ZN6impala13LikePredicate9LikeCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE", true)); - db.addBuiltin(ScalarFunction.createBuiltin( - Operator.ILIKE.name(), Lists.<Type>newArrayList(Type.STRING, Type.STRING), - false, Type.BOOLEAN, "_ZN6impala13LikePredicate4LikeEPN10impala_udf15FunctionContextERKNS1_9StringValES6_", - "_ZN6impala13LikePredicate12ILikePrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE", - "_ZN6impala13LikePredicate9LikeCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE", true)); - db.addBuiltin(ScalarFunction.createBuiltin( - Operator.RLIKE.name(), Lists.<Type>newArrayList(Type.STRING, Type.STRING), - false, Type.BOOLEAN, "_ZN6impala13LikePredicate5RegexEPN10impala_udf15FunctionContextERKNS1_9StringValES6_", - "_ZN6impala13LikePredicate12RegexPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE", - "_ZN6impala13LikePredicate10RegexCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE", true)); - db.addBuiltin(ScalarFunction.createBuiltin( - Operator.REGEXP.name(), Lists.<Type>newArrayList(Type.STRING, Type.STRING), - false, Type.BOOLEAN, "_ZN6impala13LikePredicate5RegexEPN10impala_udf15FunctionContextERKNS1_9StringValES6_", - "_ZN6impala13LikePredicate12RegexPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE", - "_ZN6impala13LikePredicate10RegexCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE", true)); - db.addBuiltin(ScalarFunction.createBuiltin( - Operator.IREGEXP.name(), Lists.<Type>newArrayList(Type.STRING, Type.STRING), - false, Type.BOOLEAN, "_ZN6impala13LikePredicate5RegexEPN10impala_udf15FunctionContextERKNS1_9StringValES6_", - "_ZN6impala13LikePredicate13IRegexPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE", - "_ZN6impala13LikePredicate10RegexCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE", true)); - } - - private final Operator op_; - - public LikePredicate(Operator op, Expr e1, Expr e2) { - super(); - this.op_ = op; - Preconditions.checkNotNull(e1); - children_.add(e1); - Preconditions.checkNotNull(e2); - children_.add(e2); - } - - /** - * Copy c'tor used in clone(). - */ - public LikePredicate(LikePredicate other) { - super(other); - op_ = other.op_; - } - - @Override - public boolean equals(Object obj) { - if (!super.equals(obj)) return false; - return ((LikePredicate) obj).op_ == op_; - } - - @Override - public String toSqlImpl() { - return getChild(0).toSql() + " " + op_.toString() + " " + getChild(1).toSql(); - } - - @Override - protected void toThrift(TExprNode msg) { - msg.node_type = TExprNodeType.FUNCTION_CALL; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - if (isAnalyzed_) return; - super.analyze(analyzer); - if (!getChild(0).getType().isStringType() && !getChild(0).getType().isNull()) { - throw new AnalysisException( - "left operand of " + op_.toString() + " must be of type STRING: " + toSql()); - } - if (!getChild(1).getType().isStringType() && !getChild(1).getType().isNull()) { - throw new AnalysisException( - "right operand of " + op_.toString() + " must be of type STRING: " + toSql()); - } - - fn_ = getBuiltinFunction(analyzer, op_.toString(), collectChildReturnTypes(), - CompareMode.IS_NONSTRICT_SUPERTYPE_OF); - Preconditions.checkState(fn_ != null); - Preconditions.checkState(fn_.getReturnType().isBoolean()); - - if (getChild(1).isLiteral() && !getChild(1).isNullLiteral() - && (op_ == Operator.RLIKE || op_ == Operator.REGEXP || op_ == Operator.IREGEXP)) { - // let's make sure the pattern works - // TODO: this checks that it's a Java-supported regex, but the syntax supported - // by the backend is Posix; add a call to the backend to check the re syntax - try { - Pattern.compile(((StringLiteral) getChild(1)).getValue()); - } catch (PatternSyntaxException e) { - throw new AnalysisException( - "invalid regular expression in '" + this.toSql() + "'"); - } - } - castForFunctionCall(false); - - if (hasChildCosts()) { - if (getChild(1).isLiteral() && !getChild(1).isNullLiteral() && - Pattern.matches("[%_]*[^%_]*[%_]*", ((StringLiteral) getChild(1)).getValue())) { - // This pattern only has wildcards as leading or trailing character, - // so it is linear. - evalCost_ = getChildCosts() + - (float) (getAvgStringLength(getChild(0)) + getAvgStringLength(getChild(1)) * - BINARY_PREDICATE_COST) + LIKE_COST; - } else { - // This pattern is more expensive, so calculate its cost as quadratic. - evalCost_ = getChildCosts() + - (float) (getAvgStringLength(getChild(0)) * getAvgStringLength(getChild(1)) * - BINARY_PREDICATE_COST) + LIKE_COST; - } - } - } - - @Override - public Expr clone() { return new LikePredicate(this); } - - public Operator getOp() { return op_; } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/LimitElement.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/LimitElement.java b/fe/src/main/java/com/cloudera/impala/analysis/LimitElement.java deleted file mode 100644 index 4de9501..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/LimitElement.java +++ /dev/null @@ -1,183 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.InternalException; -import com.cloudera.impala.service.FeSupport; -import com.cloudera.impala.thrift.TColumnValue; -import com.google.common.base.Preconditions; - -/** - * Combination of limit and offset expressions. - */ -class LimitElement { - ///////////////////////////////////////// - // BEGIN: Members that need to be reset() - - private final Expr limitExpr_; - private final Expr offsetExpr_; - private long limit_; - private long offset_; - private boolean isAnalyzed_; - - // END: Members that need to be reset() - ///////////////////////////////////////// - - /** - * Constructs the LimitElement. - * @param limitExpr The limit expression. May be null if there is no LIMIT clause. - * @param offsetExpr The offset expression. May be null if there is no OFFSET clause. - */ - public LimitElement(Expr limitExpr, Expr offsetExpr) { - this.limitExpr_ = limitExpr; - this.offsetExpr_ = offsetExpr; - isAnalyzed_ = false; - limit_ = -1; - offset_ = 0; - } - - /** - * Copy c'tor used in clone(). - */ - protected LimitElement(LimitElement other) { - limitExpr_ = (other.limitExpr_ != null) ? other.limitExpr_.clone() : null; - offsetExpr_ = (other.offsetExpr_ != null) ? other.offsetExpr_.clone() : null; - limit_ = other.limit_; - offset_ = other.offset_; - isAnalyzed_ = other.isAnalyzed_; - } - - public Expr getLimitExpr() { return limitExpr_; } - public Expr getOffsetExpr() { return offsetExpr_; } - - /** - * Returns the integer limit, evaluated from the limit expression. Must call analyze() - * first. If no limit was set, then -1 is returned. - */ - public long getLimit() { - Preconditions.checkState(isAnalyzed_); - return limit_; - } - - public boolean hasLimit() { - Preconditions.checkState(isAnalyzed_); - return limit_ != -1; - } - - /** - * Returns the integer offset, evaluated from the offset expression. Must call - * analyze() first. If no offsetExpr exists, then 0 (the default offset) is returned. - */ - public long getOffset() { - Preconditions.checkState(isAnalyzed_); - return offset_; - } - - public String toSql() { - StringBuilder sb = new StringBuilder(); - if (limitExpr_ != null) { - sb.append(" LIMIT "); - sb.append(limitExpr_.toSql()); - } - // Don't add the offset if it is the default value. However, we do print it if it - // hasn't been analyzed yet because we need to output the expression used in errors. - if (offsetExpr_ != null && (offset_ != 0 || !isAnalyzed_)) { - sb.append(" OFFSET "); - sb.append(offsetExpr_.toSql()); - } - return sb.toString(); - } - - public void analyze(Analyzer analyzer) throws AnalysisException { - isAnalyzed_ = true; - if (limitExpr_ != null) { - if (!limitExpr_.isConstant()) { - throw new AnalysisException("LIMIT expression must be a constant expression: " + - limitExpr_.toSql()); - } - - limitExpr_.analyze(analyzer); - if (!limitExpr_.getType().isIntegerType()) { - throw new AnalysisException("LIMIT expression must be an integer type but is '" + - limitExpr_.getType() + "': " + limitExpr_.toSql()); - } - limit_ = evalIntegerExpr(analyzer, limitExpr_, "LIMIT"); - } - if (limit_ == 0) analyzer.setHasEmptyResultSet(); - - if (offsetExpr_ != null) { - if (!offsetExpr_.isConstant()) { - throw new AnalysisException("OFFSET expression must be a constant expression: " + - offsetExpr_.toSql()); - } - - offsetExpr_.analyze(analyzer); - if (!offsetExpr_.getType().isIntegerType()) { - throw new AnalysisException("OFFSET expression must be an integer type but " + - "is '" + offsetExpr_.getType() + "': " + offsetExpr_.toSql()); - } - offset_ = evalIntegerExpr(analyzer, offsetExpr_, "OFFSET"); - } - } - - /** - * Evaluations an expression to a non-zero integral value, returned as a long. Throws - * if the expression cannot be evaluated, if the value evaluates to null, or if the - * result is negative. The 'name' parameter is used in exception messages, e.g. - * "LIMIT expression evaluates to NULL". - */ - private static long evalIntegerExpr(Analyzer analyzer, Expr expr, String name) - throws AnalysisException { - TColumnValue val = null; - try { - val = FeSupport.EvalConstExpr(expr, analyzer.getQueryCtx()); - } catch (InternalException e) { - throw new AnalysisException("Failed to evaluate expr: " + expr.toSql(), e); - } - long value; - if (val.isSetLong_val()) { - value = val.getLong_val(); - } else if (val.isSetInt_val()) { - value = val.getInt_val(); - } else if (val.isSetShort_val()) { - value = val.getShort_val(); - } else if (val.isSetByte_val()) { - value = val.getByte_val(); - } else { - throw new AnalysisException(name + " expression evaluates to NULL: " + - expr.toSql()); - } - if (value < 0) { - throw new AnalysisException(name + " must be a non-negative integer: " + - expr.toSql() + " = " + value); - } - return value; - } - - @Override - public LimitElement clone() { return new LimitElement(this); } - - public void reset() { - isAnalyzed_ = false; - limit_ = -1; - offset_ = 0; - if (limitExpr_ != null) limitExpr_.reset(); - if (offsetExpr_ != null) offsetExpr_.reset(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java b/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java deleted file mode 100644 index f5eedbb..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java +++ /dev/null @@ -1,242 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.math.BigDecimal; -import java.math.BigInteger; - -import com.cloudera.impala.catalog.ScalarType; -import com.cloudera.impala.catalog.Type; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.InternalException; -import com.cloudera.impala.common.NotImplementedException; -import com.cloudera.impala.service.FeSupport; -import com.cloudera.impala.thrift.TColumnValue; -import com.cloudera.impala.thrift.TExprNode; -import com.cloudera.impala.thrift.TQueryCtx; -import com.google.common.base.Preconditions; - -/** - * Representation of a literal expression. Literals are comparable to allow - * ordering of HdfsPartitions whose partition-key values are represented as literals. - */ -public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr> { - - public LiteralExpr() { - numDistinctValues_ = 1; - } - - /** - * Copy c'tor used in clone(). - */ - protected LiteralExpr(LiteralExpr other) { - super(other); - } - - /** - * Returns an analyzed literal of 'type'. Returns null for types that do not have a - * LiteralExpr subclass, e.g. TIMESTAMP. - */ - public static LiteralExpr create(String value, Type type) throws AnalysisException { - Preconditions.checkArgument(type.isValid()); - LiteralExpr e = null; - switch (type.getPrimitiveType()) { - case NULL_TYPE: - e = new NullLiteral(); - break; - case BOOLEAN: - e = new BoolLiteral(value); - break; - case TINYINT: - case SMALLINT: - case INT: - case BIGINT: - case FLOAT: - case DOUBLE: - case DECIMAL: - e = new NumericLiteral(value, type); - break; - case STRING: - case VARCHAR: - case CHAR: - e = new StringLiteral(value); - break; - case DATE: - case DATETIME: - case TIMESTAMP: - // TODO: we support TIMESTAMP but no way to specify it in SQL. - return null; - default: - Preconditions.checkState(false, - String.format("Literals of type '%s' not supported.", type.toSql())); - } - e.analyze(null); - // Need to cast since we cannot infer the type from the value. e.g. value - // can be parsed as tinyint but we need a bigint. - return (LiteralExpr) e.uncheckedCastTo(type); - } - - /** - * Returns an analyzed literal from the thrift object. - */ - public static LiteralExpr fromThrift(TExprNode exprNode, Type colType) { - try { - LiteralExpr result = null; - switch (exprNode.node_type) { - case FLOAT_LITERAL: - result = LiteralExpr.create( - Double.toString(exprNode.float_literal.value), colType); - break; - case DECIMAL_LITERAL: - byte[] bytes = exprNode.decimal_literal.getValue(); - BigDecimal val = new BigDecimal(new BigInteger(bytes)); - ScalarType decimalType = (ScalarType) colType; - // We store the decimal as the unscaled bytes. Need to adjust for the scale. - val = val.movePointLeft(decimalType.decimalScale()); - result = new NumericLiteral(val, colType); - break; - case INT_LITERAL: - result = LiteralExpr.create( - Long.toString(exprNode.int_literal.value), colType); - break; - case STRING_LITERAL: - result = LiteralExpr.create(exprNode.string_literal.value, colType); - break; - case BOOL_LITERAL: - result = LiteralExpr.create( - Boolean.toString(exprNode.bool_literal.value), colType); - break; - case NULL_LITERAL: - return NullLiteral.create(colType); - default: - throw new UnsupportedOperationException("Unsupported partition key type: " + - exprNode.node_type); - } - Preconditions.checkNotNull(result); - result.analyze(null); - return result; - } catch (Exception e) { - throw new IllegalStateException("Error creating LiteralExpr: ", e); - } - } - - // Returns the string representation of the literal's value. Used when passing - // literal values to the metastore rather than to Impala backends. This is similar to - // the toSql() method, but does not perform any formatting of the string values. Neither - // method unescapes string values. - public abstract String getStringValue(); - - // Swaps the sign of numeric literals. - // Throws for non-numeric literals. - public void swapSign() throws NotImplementedException { - throw new NotImplementedException("swapSign() only implemented for numeric" + - "literals"); - } - - /** - * Evaluates the given constant expr and returns its result as a LiteralExpr. - * Assumes expr has been analyzed. Returns constExpr if is it already a LiteralExpr. - * Returns null for types that do not have a LiteralExpr subclass, e.g. TIMESTAMP. - * TODO: Support non-scalar types. - */ - public static LiteralExpr create(Expr constExpr, TQueryCtx queryCtx) - throws AnalysisException { - Preconditions.checkState(constExpr.isConstant()); - Preconditions.checkState(constExpr.getType().isValid()); - if (constExpr instanceof LiteralExpr) return (LiteralExpr) constExpr; - - TColumnValue val = null; - try { - val = FeSupport.EvalConstExpr(constExpr, queryCtx); - } catch (InternalException e) { - throw new AnalysisException(String.format("Failed to evaluate expr '%s'", - constExpr.toSql()), e); - } - - LiteralExpr result = null; - switch (constExpr.getType().getPrimitiveType()) { - case NULL_TYPE: - result = new NullLiteral(); - break; - case BOOLEAN: - if (val.isBool_val()) result = new BoolLiteral(val.bool_val); - break; - case TINYINT: - if (val.isSetByte_val()) { - result = new NumericLiteral(BigDecimal.valueOf(val.byte_val)); - } - break; - case SMALLINT: - if (val.isSetShort_val()) { - result = new NumericLiteral(BigDecimal.valueOf(val.short_val)); - } - break; - case INT: - if (val.isSetInt_val()) { - result = new NumericLiteral(BigDecimal.valueOf(val.int_val)); - } - break; - case BIGINT: - if (val.isSetLong_val()) { - result = new NumericLiteral(BigDecimal.valueOf(val.long_val)); - } - break; - case FLOAT: - case DOUBLE: - if (val.isSetDouble_val()) { - result = - new NumericLiteral(new BigDecimal(val.double_val), constExpr.getType()); - } - break; - case DECIMAL: - if (val.isSetString_val()) { - result = - new NumericLiteral(new BigDecimal(val.string_val), constExpr.getType()); - } - break; - case STRING: - case VARCHAR: - case CHAR: - if (val.isSetString_val()) result = new StringLiteral(val.string_val); - break; - case DATE: - case DATETIME: - case TIMESTAMP: - return null; - default: - Preconditions.checkState(false, - String.format("Literals of type '%s' not supported.", - constExpr.getType().toSql())); - } - // None of the fields in the thrift struct were set indicating a NULL. - if (result == null) result = new NullLiteral(); - - result.analyze(null); - return (LiteralExpr)result; - } - - // Order NullLiterals based on the SQL ORDER BY default behavior: NULLS LAST. - @Override - public int compareTo(LiteralExpr other) { - if (this instanceof NullLiteral && other instanceof NullLiteral) return 0; - if (this instanceof NullLiteral) return -1; - if (other instanceof NullLiteral) return 1; - if (getClass() != other.getClass()) return -1; - return 0; - } -} http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/LoadDataStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/LoadDataStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/LoadDataStmt.java deleted file mode 100644 index 109d70a..0000000 --- a/fe/src/main/java/com/cloudera/impala/analysis/LoadDataStmt.java +++ /dev/null @@ -1,241 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloudera.impala.analysis; - -import java.io.FileNotFoundException; -import java.io.IOException; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.S3AFileSystem; -import org.apache.hadoop.hdfs.DistributedFileSystem; - -import com.cloudera.impala.authorization.Privilege; -import com.cloudera.impala.catalog.HdfsFileFormat; -import com.cloudera.impala.catalog.HdfsPartition; -import com.cloudera.impala.catalog.HdfsTable; -import com.cloudera.impala.catalog.Table; -import com.cloudera.impala.common.AnalysisException; -import com.cloudera.impala.common.FileSystemUtil; -import com.cloudera.impala.thrift.ImpalaInternalServiceConstants; -import com.cloudera.impala.thrift.TLoadDataReq; -import com.cloudera.impala.thrift.TTableName; -import com.cloudera.impala.util.TAccessLevelUtil; -import com.cloudera.impala.util.FsPermissionChecker; -import com.google.common.base.Preconditions; - -/* - * Represents a LOAD DATA statement for moving data into an existing table: - * LOAD DATA INPATH 'filepath' [OVERWRITE] INTO TABLE <table name> - * [PARTITION (partcol1=val1, partcol2=val2 ...)] - * - * The LOAD DATA operation supports loading (moving) a single file or all files in a - * given source directory to a table or partition location. If OVERWRITE is true, all - * exiting files in the destination will be removed before moving the new data in. - * If OVERWRITE is false, existing files will be preserved. If there are any file name - * conflicts, the new files will be uniquified by inserting a UUID into the file name - * (preserving the extension). - * Loading hidden files is not supported and any hidden files in the source or - * destination are preserved, even if OVERWRITE is true. - */ -public class LoadDataStmt extends StatementBase { - private final TableName tableName_; - private final HdfsUri sourceDataPath_; - private final PartitionSpec partitionSpec_; - private final boolean overwrite_; - - // Set during analysis - private String dbName_; - - public LoadDataStmt(TableName tableName, HdfsUri sourceDataPath, boolean overwrite, - PartitionSpec partitionSpec) { - Preconditions.checkNotNull(tableName); - Preconditions.checkNotNull(sourceDataPath); - this.tableName_ = tableName; - this.sourceDataPath_ = sourceDataPath; - this.overwrite_ = overwrite; - this.partitionSpec_ = partitionSpec; - } - - public String getTbl() { - return tableName_.getTbl(); - } - - public String getDb() { - Preconditions.checkNotNull(dbName_); - return dbName_; - } - - /* - * Print SQL syntax corresponding to this node. - * @see com.cloudera.impala.parser.ParseNode#toSql() - */ - @Override - public String toSql() { - StringBuilder sb = new StringBuilder("LOAD DATA INPATH '"); - sb.append(sourceDataPath_ + "' "); - if (overwrite_) sb.append("OVERWRITE "); - sb.append("INTO TABLE " + tableName_.toString()); - if (partitionSpec_ != null) sb.append(" " + partitionSpec_.toSql()); - return sb.toString(); - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException { - dbName_ = analyzer.getTargetDbName(tableName_); - Table table = analyzer.getTable(tableName_, Privilege.INSERT); - if (!(table instanceof HdfsTable)) { - throw new AnalysisException("LOAD DATA only supported for HDFS tables: " + - dbName_ + "." + getTbl()); - } - - // Analyze the partition spec, if one was specified. - if (partitionSpec_ != null) { - partitionSpec_.setTableName(tableName_); - partitionSpec_.setPartitionShouldExist(); - partitionSpec_.setPrivilegeRequirement(Privilege.INSERT); - partitionSpec_.analyze(analyzer); - } else { - if (table.getMetaStoreTable().getPartitionKeysSize() > 0) { - throw new AnalysisException("Table is partitioned but no partition spec was " + - "specified: " + dbName_ + "." + getTbl()); - } - } - analyzePaths(analyzer, (HdfsTable) table); - } - - /** - * Check to see if Impala has the necessary permissions to access the source and dest - * paths for this LOAD statement (which maps onto a sequence of file move operations, - * with the requisite permission requirements), and check to see if all files to be - * moved are in format that Impala understands. Errors are raised as AnalysisExceptions. - */ - private void analyzePaths(Analyzer analyzer, HdfsTable hdfsTable) - throws AnalysisException { - // The user must have permission to access the source location. Since the files will - // be moved from this location, the user needs to have all permission. - sourceDataPath_.analyze(analyzer, Privilege.ALL); - - // Catch all exceptions thrown by accessing files, and rethrow as AnalysisExceptions. - try { - Path source = sourceDataPath_.getPath(); - FileSystem fs = source.getFileSystem(FileSystemUtil.getConfiguration()); - if (!(fs instanceof DistributedFileSystem) && !(fs instanceof S3AFileSystem)) { - throw new AnalysisException(String.format("INPATH location '%s' " + - "must point to an HDFS or S3A filesystem.", sourceDataPath_)); - } - if (!fs.exists(source)) { - throw new AnalysisException(String.format( - "INPATH location '%s' does not exist.", sourceDataPath_)); - } - - // If the source file is a directory, we must be able to read from and write to - // it. If the source file is a file, we must be able to read from it, and write to - // its parent directory (in order to delete the file as part of the move operation). - FsPermissionChecker checker = FsPermissionChecker.getInstance(); - - if (fs.isDirectory(source)) { - if (FileSystemUtil.getTotalNumVisibleFiles(source) == 0) { - throw new AnalysisException(String.format( - "INPATH location '%s' contains no visible files.", sourceDataPath_)); - } - if (FileSystemUtil.containsVisibleSubdirectory(source)) { - throw new AnalysisException(String.format( - "INPATH location '%s' cannot contain non-hidden subdirectories.", - sourceDataPath_)); - } - if (!checker.getPermissions(fs, source).checkPermissions( - FsAction.READ_WRITE)) { - throw new AnalysisException(String.format("Unable to LOAD DATA from %s " + - "because Impala does not have READ and WRITE permissions on this directory", - source)); - } - } else { - // INPATH names a file. - if (FileSystemUtil.isHiddenFile(source.getName())) { - throw new AnalysisException(String.format( - "INPATH location '%s' points to a hidden file.", source)); - } - - if (!checker.getPermissions(fs, source.getParent()).checkPermissions( - FsAction.WRITE)) { - throw new AnalysisException(String.format("Unable to LOAD DATA from %s " + - "because Impala does not have WRITE permissions on its parent " + - "directory %s", source, source.getParent())); - } - - if (!checker.getPermissions(fs, source).checkPermissions( - FsAction.READ)) { - throw new AnalysisException(String.format("Unable to LOAD DATA from %s " + - "because Impala does not have READ permissions on this file", source)); - } - } - - String noWriteAccessErrorMsg = String.format("Unable to LOAD DATA into " + - "target table (%s) because Impala does not have WRITE access to HDFS " + - "location: ", hdfsTable.getFullName()); - - HdfsPartition partition; - String location; - if (partitionSpec_ != null) { - partition = hdfsTable.getPartition(partitionSpec_.getPartitionSpecKeyValues()); - location = partition.getLocation(); - if (!TAccessLevelUtil.impliesWriteAccess(partition.getAccessLevel())) { - throw new AnalysisException(noWriteAccessErrorMsg + location); - } - } else { - // "default" partition - partition = hdfsTable.getPartitionMap().get( - ImpalaInternalServiceConstants.DEFAULT_PARTITION_ID); - location = hdfsTable.getLocation(); - if (!hdfsTable.hasWriteAccess()) { - throw new AnalysisException(noWriteAccessErrorMsg + hdfsTable.getLocation()); - } - } - Preconditions.checkNotNull(partition); - - // Verify the files being loaded are supported. - for (FileStatus fStatus: fs.listStatus(source)) { - if (fs.isDirectory(fStatus.getPath())) continue; - StringBuilder errorMsg = new StringBuilder(); - HdfsFileFormat fileFormat = partition.getInputFormatDescriptor().getFileFormat(); - if (!fileFormat.isFileCompressionTypeSupported(fStatus.getPath().toString(), - errorMsg)) { - throw new AnalysisException(errorMsg.toString()); - } - } - } catch (FileNotFoundException e) { - throw new AnalysisException("File not found: " + e.getMessage(), e); - } catch (IOException e) { - throw new AnalysisException("Error accessing filesystem: " + e.getMessage(), e); - } - } - - public TLoadDataReq toThrift() { - TLoadDataReq loadDataReq = new TLoadDataReq(); - loadDataReq.setTable_name(new TTableName(getDb(), getTbl())); - loadDataReq.setSource_path(sourceDataPath_.toString()); - loadDataReq.setOverwrite(overwrite_); - if (partitionSpec_ != null) { - loadDataReq.setPartition_spec(partitionSpec_.toThrift()); - } - return loadDataReq; - } -}
