IMPALA-5152: Introduce metadata loading phase
Reworks the collection and loading of missing metadata
when compiling a statement. Introduces a new
metadata-loading phase between parsing and analysis.
Summary of the new compilation flow:
1. Parse statement.
2. Collect all table references from the parsed
statement and generate a list of tables that need
to be loaded for analysis to succeed.
3. Request missing metadata and wait for it to arrive.
As views become loaded we expand the set of required
tables based on the view definitions.
This step populates a statement-local table cache
that contains all loaded tables relevant to the
statement.
4. Create a new Analyzer with the table cache and
analyze the statement. During analysis only the
table cache is consulted for table metadata, the
ImpaladCatalog is not used for that purpose anymore.
5. Authorize the statement.
6. Plan generation as usual.
The intent of the existing code was to collect all tables
missing metadata during analysis, load the metadata, and then
re-analyze the statement (and repeat those steps until all
metadata is loaded).
Unfortunately, the relevant code was hard-to-follow, subtle
and not well tested, and therefore it was broken in several
ways over the course of time. For example, the introduction
of path analysis for nested types subtly broke the intended
behavior, and there are other similar examples.
The serial table loading observed in the JIRA was caused by the
following code in the resolution of table references:
for (all path interpretations) {
try {
// Try to resolve the path; might call getTable() which
// throws for nonexistent tables.
} catch (AnalysisException e) {
if (analyzer.hasMissingTbls()) throw e;
}
}
The following example illustrates the problem:
SELECT * FROM a.b, x.y
When resolving the path "a.b" we consider that "a" could be a
database or a table. Similarly, "b" could be a table or a
nested collection.
If the path resolution for "a.b" adds a missing table entry,
then the path resolution for "x.y" could exit prematurely,
without trying the other path interpretations that would
lead to adding the expected missing table. So effectively,
the tables end up being loaded one-by-one.
Testing:
- A core/hdfs run succeeded
- No new tests were added because the existing functional tests
provide good coverage of various metadata loading scenarios.
- The issue reported in IMPALA-5152 is basically impossible now.
Adding FE unit tests for that bug specifically would require
ugly changes to the new code to enable such testing.
Change-Id: I68d32d5acd4a6f6bc6cedb05e6cc5cf604d24a55
Reviewed-on: http://gerrit.cloudera.org:8080/8958
Reviewed-by: Alex Behm <alex.b...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/8ea1ce87
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/8ea1ce87
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/8ea1ce87
Branch: refs/heads/master
Commit: 8ea1ce87e2150c843b4da15f9d42b87006e6ffca
Parents: d462178
Author: Alex Behm <alex.b...@cloudera.com>
Authored: Fri Apr 7 09:58:40 2017 -0700
Committer: Impala Public Jenkins <impala-public-jenk...@gerrit.cloudera.org>
Committed: Thu Feb 22 06:55:03 2018 +0000
----------------------------------------------------------------------
.../apache/impala/analysis/AlterTableStmt.java | 7 +
.../apache/impala/analysis/AnalysisContext.java | 170 ++++++-----
.../org/apache/impala/analysis/Analyzer.java | 133 ++++-----
.../impala/analysis/AuthorizationStmt.java | 1 +
.../impala/analysis/ComputeStatsStmt.java | 5 +
.../apache/impala/analysis/CreateDbStmt.java | 2 -
.../impala/analysis/CreateFunctionStmtBase.java | 2 +-
.../analysis/CreateOrAlterViewStmtBase.java | 6 +
.../analysis/CreateTableAsSelectStmt.java | 23 +-
.../impala/analysis/CreateTableLikeStmt.java | 13 +-
.../apache/impala/analysis/CreateTableStmt.java | 5 +
.../apache/impala/analysis/DescribeDbStmt.java | 1 +
.../impala/analysis/DescribeTableStmt.java | 7 +-
.../apache/impala/analysis/DropStatsStmt.java | 11 +-
.../impala/analysis/DropTableOrViewStmt.java | 13 +-
.../org/apache/impala/analysis/FromClause.java | 39 +--
.../impala/analysis/GrantRevokePrivStmt.java | 6 +
.../org/apache/impala/analysis/InsertStmt.java | 48 ++--
.../apache/impala/analysis/LimitElement.java | 7 +-
.../apache/impala/analysis/LoadDataStmt.java | 8 +-
.../org/apache/impala/analysis/ModifyStmt.java | 34 ++-
.../java/org/apache/impala/analysis/Path.java | 23 ++
.../apache/impala/analysis/PrivilegeSpec.java | 7 +-
.../org/apache/impala/analysis/QueryStmt.java | 35 ++-
.../impala/analysis/ResetMetadataStmt.java | 7 +
.../org/apache/impala/analysis/SelectStmt.java | 25 +-
.../org/apache/impala/analysis/SetStmt.java | 1 +
.../impala/analysis/ShowCreateFunctionStmt.java | 4 +-
.../impala/analysis/ShowCreateTableStmt.java | 10 +-
.../apache/impala/analysis/ShowFilesStmt.java | 11 +-
.../impala/analysis/ShowFunctionsStmt.java | 1 +
.../impala/analysis/ShowGrantRoleStmt.java | 8 +
.../apache/impala/analysis/ShowStatsStmt.java | 12 +-
.../apache/impala/analysis/ShowTablesStmt.java | 1 +
.../apache/impala/analysis/StatementBase.java | 9 +-
.../impala/analysis/StmtMetadataLoader.java | 280 +++++++++++++++++++
.../apache/impala/analysis/TruncateStmt.java | 9 +-
.../org/apache/impala/analysis/UnionStmt.java | 43 ++-
.../org/apache/impala/analysis/WithClause.java | 40 ++-
.../java/org/apache/impala/catalog/Catalog.java | 26 +-
.../apache/impala/catalog/ImpaladCatalog.java | 33 +--
.../apache/impala/common/AnalysisException.java | 4 +
.../apache/impala/common/ImpalaException.java | 4 +
.../java/org/apache/impala/planner/Planner.java | 17 +-
.../apache/impala/planner/PlannerContext.java | 14 +-
.../impala/planner/SingleNodePlanner.java | 11 +-
.../impala/service/CatalogOpExecutor.java | 10 +-
.../org/apache/impala/service/FeSupport.java | 13 +-
.../org/apache/impala/service/Frontend.java | 193 +++----------
.../org/apache/impala/service/MetadataOp.java | 38 ++-
.../org/apache/impala/util/EventSequence.java | 3 +
.../impala/analysis/AnalyzeAuthStmtsTest.java | 81 +++---
.../apache/impala/analysis/AnalyzeDDLTest.java | 47 ++--
.../impala/analysis/AnalyzeExprsTest.java | 53 ++--
.../impala/analysis/AnalyzeModifyStmtsTest.java | 15 +-
.../impala/analysis/AnalyzeStmtsTest.java | 36 +--
.../impala/analysis/AnalyzeSubqueriesTest.java | 5 +-
.../impala/analysis/AnalyzeUpsertStmtTest.java | 6 +-
.../apache/impala/analysis/AnalyzerTest.java | 4 +-
.../apache/impala/analysis/AuditingTest.java | 27 +-
.../impala/analysis/AuthorizationTest.java | 173 +++++-------
.../org/apache/impala/analysis/ExprNdvTest.java | 23 +-
.../impala/analysis/ExprRewriteRulesTest.java | 57 ++--
.../impala/analysis/ExprRewriterTest.java | 31 +-
.../impala/analysis/StmtMetadataLoaderTest.java | 180 ++++++++++++
.../org/apache/impala/analysis/ToSqlTest.java | 23 +-
.../apache/impala/common/FrontendTestBase.java | 128 ++++++---
.../org/apache/impala/service/FrontendTest.java | 18 +-
.../impala/testutil/ImpaladTestCatalog.java | 57 ++--
69 files changed, 1460 insertions(+), 937 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
b/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
index 6c7530a..75abaa7 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
@@ -17,6 +17,8 @@
package org.apache.impala.analysis;
+import java.util.List;
+
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.DataSourceTable;
import org.apache.impala.catalog.Table;
@@ -67,6 +69,11 @@ public abstract class AlterTableStmt extends StatementBase {
}
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(tableName_.toPath(), null));
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
// Resolve and analyze this table ref so we can evaluate partition
predicates.
TableRef tableRef = new TableRef(tableName_.toPath(), null,
Privilege.ALTER);
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
b/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
index 5ad97eb..c886079 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
@@ -17,12 +17,12 @@
package org.apache.impala.analysis;
-import java.io.StringReader;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.impala.analysis.StmtMetadataLoader.StmtTableCache;
import org.apache.impala.authorization.AuthorizationChecker;
import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.authorization.AuthorizeableColumn;
@@ -34,12 +34,15 @@ import org.apache.impala.catalog.Db;
import org.apache.impala.catalog.ImpaladCatalog;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
+import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.Pair;
+import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.rewrite.ExprRewriter;
import org.apache.impala.thrift.TAccessEvent;
import org.apache.impala.thrift.TLineageGraph;
import org.apache.impala.thrift.TQueryCtx;
+import org.apache.impala.thrift.TQueryOptions;
import org.apache.impala.util.EventSequence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,47 +57,39 @@ import com.google.common.collect.Maps;
*/
public class AnalysisContext {
private final static Logger LOG =
LoggerFactory.getLogger(AnalysisContext.class);
- private ImpaladCatalog catalog_;
private final TQueryCtx queryCtx_;
private final AuthorizationConfig authzConfig_;
- private final ExprRewriter customRewriter_;
-
- // Timeline of important events in the planning process, used for debugging
- // and profiling
- private final EventSequence timeline_ = new EventSequence("Planner
Timeline");
+ private final EventSequence timeline_;
- // Set in analyze()
+ // Set in analyzeAndAuthorize().
+ private ImpaladCatalog catalog_;
private AnalysisResult analysisResult_;
- public AnalysisContext(ImpaladCatalog catalog, TQueryCtx queryCtx,
- AuthorizationConfig authzConfig) {
- setCatalog(catalog);
+ // Use Hive's scheme for auto-generating column labels. Only used for
testing.
+ private boolean useHiveColLabels_;
+
+ public AnalysisContext(TQueryCtx queryCtx, AuthorizationConfig authzConfig,
+ EventSequence timeline) {
queryCtx_ = queryCtx;
authzConfig_ = authzConfig;
- customRewriter_ = null;
+ timeline_ = timeline;
}
- /**
- * C'tor with a custom ExprRewriter for testing.
- */
- protected AnalysisContext(ImpaladCatalog catalog, TQueryCtx queryCtx,
- AuthorizationConfig authzConfig, ExprRewriter rewriter) {
- setCatalog(catalog);
- queryCtx_ = queryCtx;
- authzConfig_ = authzConfig;
- customRewriter_ = rewriter;
+ public ImpaladCatalog getCatalog() { return catalog_; }
+ public TQueryCtx getQueryCtx() { return queryCtx_; }
+ public TQueryOptions getQueryOptions() {
+ return queryCtx_.client_request.query_options;
}
+ public String getUser() { return queryCtx_.session.connected_user; }
- // Catalog may change between analysis attempts (e.g. when missing tables
are loaded).
- public void setCatalog(ImpaladCatalog catalog) {
- Preconditions.checkNotNull(catalog);
- catalog_ = catalog;
+ public void setUseHiveColLabels(boolean b) {
+ Preconditions.checkState(RuntimeEnv.INSTANCE.isTestEnv());
+ useHiveColLabels_ = b;
}
static public class AnalysisResult {
private StatementBase stmt_;
private Analyzer analyzer_;
- private EventSequence timeline_;
private boolean userHasProfileAccess_ = true;
public boolean isAlterTableStmt() { return stmt_ instanceof
AlterTableStmt; }
@@ -184,6 +179,27 @@ public class AnalysisContext {
return isInsertStmt();
}
+ /**
+ * Returns true for statements that may produce several privilege requests
of
+ * hierarchical nature, e.g., table/column.
+ */
+ public boolean isHierarchicalAuthStmt() {
+ return isQueryStmt() || isInsertStmt() || isUpdateStmt() ||
isDeleteStmt()
+ || isCreateTableAsSelectStmt() || isCreateViewStmt() ||
isAlterViewStmt();
+ }
+
+ /**
+ * Returns true for statements that may produce a single column-level
privilege
+ * request without a request at the table level.
+ * Example: USE functional; ALTER TABLE allcomplextypes.int_array_col
[...];
+ * The path 'allcomplextypes.int_array_col' table ref path resolves to
+ * a column, so a column-level privilege request is registered.
+ */
+ public boolean isSingleColumnPrivStmt() {
+ return isDescribeTableStmt() || isResetMetadataStmt() || isUseStmt()
+ || isShowTablesStmt() || isAlterTableStmt();
+ }
+
public AlterTableStmt getAlterTableStmt() {
Preconditions.checkState(isAlterTableStmt());
return (AlterTableStmt) stmt_;
@@ -335,7 +351,6 @@ public class AnalysisContext {
public StatementBase getStmt() { return stmt_; }
public Analyzer getAnalyzer() { return analyzer_; }
- public EventSequence getTimeline() { return timeline_; }
public Set<TAccessEvent> getAccessEvents() { return
analyzer_.getAccessEvents(); }
public boolean requiresSubqueryRewrite() {
return analyzer_.containsSubquery() && !(stmt_ instanceof CreateViewStmt)
@@ -352,46 +367,66 @@ public class AnalysisContext {
public boolean userHasProfileAccess() { return userHasProfileAccess_; }
}
+ public Analyzer createAnalyzer(StmtTableCache stmtTableCache) {
+ Analyzer result = new Analyzer(stmtTableCache, queryCtx_, authzConfig_);
+ result.setUseHiveColLabels(useHiveColLabels_);
+ return result;
+ }
+
/**
- * Parse and analyze 'stmt'. If 'stmt' is a nested query (i.e. query that
- * contains subqueries), it is also rewritten by performing subquery
unnesting.
- * The transformed stmt is then re-analyzed in a new analysis context.
- *
- * The result of analysis can be retrieved by calling
- * getAnalysisResult().
- *
- * @throws AnalysisException
- * On any other error, including parsing errors. Also thrown when
any
- * missing tables are detected as a result of running analysis.
+ * Analyzes and authorizes the given statement using the provided table
cache and
+ * authorization checker.
+ * AuthorizationExceptions take precedence over AnalysisExceptions so as not
to
+ * reveal the existence/absence of objects the user is not authorized to see.
*/
- public void analyze(String stmt) throws AnalysisException {
- Analyzer analyzer = new Analyzer(catalog_, queryCtx_, authzConfig_);
- analyze(stmt, analyzer);
+ public AnalysisResult analyzeAndAuthorize(StatementBase stmt,
+ StmtTableCache stmtTableCache, AuthorizationChecker authzChecker)
+ throws ImpalaException {
+ // TODO: Clean up the creation/setting of the analysis result.
+ analysisResult_ = new AnalysisResult();
+ analysisResult_.stmt_ = stmt;
+ catalog_ = stmtTableCache.catalog;
+
+ // Analyze statement and record exception.
+ AnalysisException analysisException = null;
+ try {
+ analyze(stmtTableCache);
+ } catch (AnalysisException e) {
+ analysisException = e;
+ }
+
+ // Authorize statement and record exception. Authorization relies on
information
+ // collected during analysis.
+ AuthorizationException authException = null;
+ try {
+ authorize(authzChecker);
+ } catch (AuthorizationException e) {
+ authException = e;
+ }
+
+ // AuthorizationExceptions take precedence over AnalysisExceptions so as
not
+ // to reveal the existence/absence of objects the user is not authorized
to see.
+ if (authException != null) throw authException;
+ if (analysisException != null) throw analysisException;
+ return analysisResult_;
}
/**
- * Parse and analyze 'stmt' using a specified Analyzer.
+ * Analyzes the statement set in 'analysisResult_' with a new Analyzer based
on the
+ * given loaded tables. Performs expr and subquery rewrites which require
re-analyzing
+ * the transformed statement.
*/
- public void analyze(String stmt, Analyzer analyzer) throws AnalysisException
{
- SqlScanner input = new SqlScanner(new StringReader(stmt));
- SqlParser parser = new SqlParser(input);
+ private void analyze(StmtTableCache stmtTableCache) throws AnalysisException
{
+ Preconditions.checkNotNull(analysisResult_);
+ Preconditions.checkNotNull(analysisResult_.stmt_);
try {
- analysisResult_ = new AnalysisResult();
- analysisResult_.analyzer_ = analyzer;
- if (analysisResult_.analyzer_ == null) {
- analysisResult_.analyzer_ = new Analyzer(catalog_, queryCtx_,
authzConfig_);
- }
- analysisResult_.timeline_ = timeline_;
- analysisResult_.stmt_ = (StatementBase) parser.parse().value;
- if (analysisResult_.stmt_ == null) return;
-
+ analysisResult_.analyzer_ = createAnalyzer(stmtTableCache);
analysisResult_.stmt_.analyze(analysisResult_.analyzer_);
boolean isExplain = analysisResult_.isExplainStmt();
// Apply expr and subquery rewrites.
boolean reAnalyze = false;
- ExprRewriter rewriter = (customRewriter_ != null) ? customRewriter_ :
- analyzer.getExprRewriter();
+ ExprRewriter rewriter = analysisResult_.analyzer_.getExprRewriter();
if (analysisResult_.requiresExprRewrite()) {
rewriter.reset();
analysisResult_.stmt_.rewriteExprs(rewriter);
@@ -414,7 +449,7 @@ public class AnalysisContext {
Lists.newArrayList(analysisResult_.stmt_.getColLabels());
// Re-analyze the stmt with a new analyzer.
- analysisResult_.analyzer_ = new Analyzer(catalog_, queryCtx_,
authzConfig_);
+ analysisResult_.analyzer_ = createAnalyzer(stmtTableCache);
analysisResult_.stmt_.reset();
analysisResult_.stmt_.analyze(analysisResult_.analyzer_);
@@ -430,8 +465,6 @@ public class AnalysisContext {
} catch (AnalysisException e) {
// Don't wrap AnalysisExceptions in another AnalysisException
throw e;
- } catch (Exception e) {
- throw new AnalysisException(parser.getErrorMsg(stmt), e);
}
}
@@ -440,16 +473,15 @@ public class AnalysisContext {
* analyze() must have already been called. Throws an AuthorizationException
if the
* user doesn't have sufficient privileges to run this statement.
*/
- public void authorize(AuthorizationChecker authzChecker)
+ private void authorize(AuthorizationChecker authzChecker)
throws AuthorizationException, InternalException {
Preconditions.checkNotNull(analysisResult_);
Analyzer analyzer = getAnalyzer();
- // Process statements for which column-level privilege requests may be
registered
- // except for DESCRIBE TABLE, REFRESH/INVALIDATE, USE or SHOW TABLES
statements.
- if (analysisResult_.isQueryStmt() || analysisResult_.isInsertStmt() ||
- analysisResult_.isUpdateStmt() || analysisResult_.isDeleteStmt() ||
- analysisResult_.isCreateTableAsSelectStmt() ||
- analysisResult_.isCreateViewStmt() ||
analysisResult_.isAlterViewStmt()) {
+ // Authorize statements that may produce several hierarchical privilege
requests.
+ // Such a statement always has a corresponding table-level privilege
request if it
+ // has column-level privilege request. The hierarchical nature requires
special
+ // logic to process correctly and efficiently.
+ if (analysisResult_.isHierarchicalAuthStmt()) {
// Map of table name to a list of privilege requests associated with
that table.
// These include both table-level and column-level privilege requests.
We use a
// LinkedHashMap to preserve the order in which requests are inserted.
@@ -492,10 +524,7 @@ public class AnalysisContext {
for (PrivilegeRequest privReq: analyzer.getPrivilegeReqs()) {
Preconditions.checkState(
!(privReq.getAuthorizeable() instanceof AuthorizeableColumn) ||
- analysisResult_.isDescribeTableStmt() ||
- analysisResult_.isResetMetadataStmt() ||
- analysisResult_.isUseStmt() ||
- analysisResult_.isShowTablesStmt());
+ analysisResult_.isSingleColumnPrivStmt());
authorizePrivilegeRequest(authzChecker, privReq);
}
}
@@ -603,7 +632,6 @@ public class AnalysisContext {
return false;
}
- public AnalysisResult getAnalysisResult() { return analysisResult_; }
- public Analyzer getAnalyzer() { return getAnalysisResult().getAnalyzer(); }
+ public Analyzer getAnalyzer() { return analysisResult_.getAnalyzer(); }
public EventSequence getTimeline() { return timeline_; }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index 5e9c788..02bcf31 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -19,7 +19,6 @@ package org.apache.impala.analysis;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
@@ -30,13 +29,13 @@ import java.util.Map;
import java.util.Set;
import org.apache.impala.analysis.Path.PathType;
+import org.apache.impala.analysis.StmtMetadataLoader.StmtTableCache;
import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.authorization.AuthorizeableTable;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.authorization.PrivilegeRequest;
import org.apache.impala.authorization.PrivilegeRequestBuilder;
import org.apache.impala.authorization.User;
-import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.DataSourceTable;
import org.apache.impala.catalog.DatabaseNotFoundException;
@@ -44,6 +43,7 @@ import org.apache.impala.catalog.Db;
import org.apache.impala.catalog.HBaseTable;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.ImpaladCatalog;
+import org.apache.impala.catalog.IncompleteTable;
import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.TableLoadingException;
@@ -57,7 +57,6 @@ import org.apache.impala.common.Pair;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.planner.PlanNode;
import org.apache.impala.rewrite.BetweenToCompoundRule;
-import org.apache.impala.rewrite.RemoveRedundantStringCast;
import org.apache.impala.rewrite.EqualityDisjunctsToInRule;
import org.apache.impala.rewrite.ExprRewriteRule;
import org.apache.impala.rewrite.ExprRewriter;
@@ -66,6 +65,7 @@ import org.apache.impala.rewrite.FoldConstantsRule;
import org.apache.impala.rewrite.NormalizeBinaryPredicatesRule;
import org.apache.impala.rewrite.NormalizeCountStarRule;
import org.apache.impala.rewrite.NormalizeExprsRule;
+import org.apache.impala.rewrite.RemoveRedundantStringCast;
import org.apache.impala.rewrite.SimplifyConditionalsRule;
import org.apache.impala.rewrite.SimplifyDistinctFromRule;
import org.apache.impala.service.FeSupport;
@@ -192,9 +192,6 @@ public class Analyzer {
// a tuple is outer/semi joined, etc. Remove the maps in favor of making
// them properties of the tuple descriptor itself.
private static class GlobalState {
- // TODO: Consider adding an "exec-env"-like global singleton that contains
the
- // catalog and authzConfig.
- public final ImpaladCatalog catalog;
public final TQueryCtx queryCtx;
public final AuthorizationConfig authzConfig;
public final DescriptorTable descTbl = new DescriptorTable();
@@ -302,10 +299,8 @@ public class Analyzer {
// Decreases the size of the scan range locations.
private final ListMap<TNetworkAddress> hostIndex = new
ListMap<TNetworkAddress>();
- // The Impalad Catalog has the latest tables from the statestore. In order
to use the
- // same version of a table in a single query, we cache all referenced
tables here.
- // TODO: Investigate what to do with other catalog objects.
- private final HashMap<TableName, Table> referencedTables_ =
Maps.newHashMap();
+ // Cache of statement-relevant table metadata populated before analysis.
+ private final StmtTableCache stmtTableCache;
// Expr rewriter for folding constants.
private final ExprRewriter constantFolder_ =
@@ -314,9 +309,9 @@ public class Analyzer {
// Expr rewriter for normalizing and rewriting expressions.
private final ExprRewriter exprRewriter_;
- public GlobalState(ImpaladCatalog catalog, TQueryCtx queryCtx,
+ public GlobalState(StmtTableCache stmtTableCache, TQueryCtx queryCtx,
AuthorizationConfig authzConfig) {
- this.catalog = catalog;
+ this.stmtTableCache = stmtTableCache;
this.queryCtx = queryCtx;
this.authzConfig = authzConfig;
this.lineageGraph = new ColumnLineageGraph();
@@ -373,9 +368,6 @@ public class Analyzer {
// that have a scalar type as destination (see registerSlotRef()).
private final Map<String, SlotDescriptor> slotPathMap_ = Maps.newHashMap();
- // Tracks the all tables/views found during analysis that were missing
metadata.
- private Set<TableName> missingTbls_ = new HashSet<TableName>();
-
// Indicates whether this analyzer/block is guaranteed to have an empty
result set
// due to a limit 0 or constant conjunct evaluating to false.
private boolean hasEmptyResultSet_ = false;
@@ -385,10 +377,10 @@ public class Analyzer {
// conjunct evaluating to false.
private boolean hasEmptySpjResultSet_ = false;
- public Analyzer(ImpaladCatalog catalog, TQueryCtx queryCtx,
+ public Analyzer(StmtTableCache stmtTableCache, TQueryCtx queryCtx,
AuthorizationConfig authzConfig) {
ancestors_ = Lists.newArrayList();
- globalState_ = new GlobalState(catalog, queryCtx, authzConfig);
+ globalState_ = new GlobalState(stmtTableCache, queryCtx, authzConfig);
user_ = new User(TSessionStateUtil.getEffectiveUser(queryCtx.session));
}
@@ -407,7 +399,6 @@ public class Analyzer {
ancestors_ = Lists.newArrayList(parentAnalyzer);
ancestors_.addAll(parentAnalyzer.ancestors_);
globalState_ = globalState;
- missingTbls_ = parentAnalyzer.missingTbls_;
user_ = parentAnalyzer.getUser();
useHiveColLabels_ = parentAnalyzer.useHiveColLabels_;
authErrorMsg_ = parentAnalyzer.authErrorMsg_;
@@ -421,7 +412,7 @@ public class Analyzer {
* global state.
*/
public static Analyzer createWithNewGlobalState(Analyzer parentAnalyzer) {
- GlobalState globalState = new
GlobalState(parentAnalyzer.globalState_.catalog,
+ GlobalState globalState = new
GlobalState(parentAnalyzer.globalState_.stmtTableCache,
parentAnalyzer.getQueryCtx(), parentAnalyzer.getAuthzConfig());
return new Analyzer(parentAnalyzer, globalState);
}
@@ -437,8 +428,6 @@ public class Analyzer {
visibleSemiJoinedTupleId_ = tid;
}
- public Set<TableName> getMissingTbls() { return missingTbls_; }
- public boolean hasMissingTbls() { return !missingTbls_.isEmpty(); }
public boolean hasAncestors() { return !ancestors_.isEmpty(); }
public Analyzer getParentAnalyzer() {
return hasAncestors() ? ancestors_.get(0) : null;
@@ -575,19 +564,17 @@ public class Analyzer {
try {
resolvedPath = resolvePath(tableRef.getPath(), PathType.TABLE_REF);
} catch (AnalysisException e) {
- if (!hasMissingTbls()) {
- // Register privilege requests to prefer reporting an authorization
error over
- // an analysis error. We should not accidentally reveal the
non-existence of a
- // table/database if the user is not authorized.
- if (rawPath.size() > 1) {
- registerPrivReq(new PrivilegeRequestBuilder()
- .onTable(rawPath.get(0), rawPath.get(1))
- .allOf(tableRef.getPrivilege()).toRequest());
- }
+ // Register privilege requests to prefer reporting an authorization
error over
+ // an analysis error. We should not accidentally reveal the
non-existence of a
+ // table/database if the user is not authorized.
+ if (rawPath.size() > 1) {
registerPrivReq(new PrivilegeRequestBuilder()
- .onTable(getDefaultDb(), rawPath.get(0))
+ .onTable(rawPath.get(0), rawPath.get(1))
.allOf(tableRef.getPrivilege()).toRequest());
}
+ registerPrivReq(new PrivilegeRequestBuilder()
+ .onTable(getDefaultDb(), rawPath.get(0))
+ .allOf(tableRef.getPrivilege()).toRequest());
throw e;
} catch (TableLoadingException e) {
throw new AnalysisException(String.format(
@@ -780,16 +767,14 @@ public class Analyzer {
candidates.clear();
// Add paths rooted at a table with an unqualified and fully-qualified
table name.
- int end = Math.min(2, rawPath.size());
- for (int tblNameIdx = 0; tblNameIdx < end; ++tblNameIdx) {
- String dbName = (tblNameIdx == 0) ? getDefaultDb() : rawPath.get(0);
- String tblName = rawPath.get(tblNameIdx);
+ List<TableName> candidateTbls = Path.getCandidateTables(rawPath,
getDefaultDb());
+ for (int tblNameIdx = 0; tblNameIdx < candidateTbls.size();
++tblNameIdx) {
+ TableName tblName = candidateTbls.get(tblNameIdx);
Table tbl = null;
try {
- tbl = getTable(dbName, tblName);
+ tbl = getTable(tblName.getDb(), tblName.getTbl());
} catch (AnalysisException e) {
- if (hasMissingTbls()) throw e;
- // Ignore other exceptions to allow path resolution to continue.
+ // Ignore to allow path resolution to continue.
}
if (tbl != null) {
candidates.add(new Path(tbl, rawPath.subList(tblNameIdx + 1,
rawPath.size())));
@@ -1318,7 +1303,8 @@ public class Analyzer {
}
public DescriptorTable getDescTbl() { return globalState_.descTbl; }
- public ImpaladCatalog getCatalog() { return globalState_.catalog; }
+ public ImpaladCatalog getCatalog() { return
globalState_.stmtTableCache.catalog; }
+ public StmtTableCache getStmtTableCache() { return
globalState_.stmtTableCache; }
public Set<String> getAliases() { return aliasMap_.keySet(); }
/**
@@ -2361,64 +2347,47 @@ public class Analyzer {
}
/**
- * Returns the Catalog Table object for the given database and table name. A
table
- * referenced for the first time is cached in
globalState_.referencedTables_. The same
- * table instance is returned for all subsequent references in the same
query.
- * Adds the table to this analyzer's "missingTbls_" and throws an
AnalysisException if
- * the table has not yet been loaded in the local catalog cache.
- * Throws an AnalysisException if the table or the db does not exist in the
Catalog.
- * This function does not register authorization requests and does not log
access events.
+ * Returns the Table for the given database and table name from the
'stmtTableCache'
+ * in the global analysis state.
+ * Throws an AnalysisException if the database or table does not exist.
+ * Throws a TableLoadingException if the registered table failed to load.
+ * Does not register authorization requests or access events.
*/
public Table getTable(String dbName, String tableName)
throws AnalysisException, TableLoadingException {
TableName tblName = new TableName(dbName, tableName);
- Table table = globalState_.referencedTables_.get(tblName);
- if (table != null) {
- // Return query-local version of table.
- Preconditions.checkState(table.isLoaded());
- return table;
- }
- try {
- table = getCatalog().getTable(dbName, tableName);
- } catch (DatabaseNotFoundException e) {
- throw new AnalysisException(DB_DOES_NOT_EXIST_ERROR_MSG + dbName);
- } catch (CatalogException e) {
- String errMsg = String.format("Failed to load metadata for table: %s",
tableName);
- // We don't want to log all AnalysisExceptions as ERROR, only failures
due to
- // TableLoadingExceptions.
- LOG.error(String.format("%s\n%s", errMsg, e.getMessage()));
- if (e instanceof TableLoadingException) throw (TableLoadingException) e;
- throw new TableLoadingException(errMsg, e);
- }
+ Table table = globalState_.stmtTableCache.tables.get(tblName);
if (table == null) {
- throw new AnalysisException(
- TBL_DOES_NOT_EXIST_ERROR_MSG + dbName + "." + tableName);
+ if (!globalState_.stmtTableCache.dbs.contains(tblName.getDb())) {
+ throw new AnalysisException(DB_DOES_NOT_EXIST_ERROR_MSG +
tblName.getDb());
+ } else {
+ throw new AnalysisException(TBL_DOES_NOT_EXIST_ERROR_MSG +
tblName.toString());
+ }
}
- if (!table.isLoaded()) {
- missingTbls_.add(new TableName(table.getDb().getName(),
table.getName()));
- throw new AnalysisException(
- "Table/view is missing metadata: " + table.getFullName());
+ Preconditions.checkState(table.isLoaded());
+ if (table instanceof IncompleteTable) {
+ // If there were problems loading this table's metadata, throw an
exception
+ // when it is accessed.
+ ImpalaException cause = ((IncompleteTable) table).getCause();
+ if (cause instanceof TableLoadingException) throw
(TableLoadingException) cause;
+ throw new TableLoadingException("Missing metadata for table: " +
tableName, cause);
}
- globalState_.referencedTables_.put(tblName, table);
return table;
}
/**
- * Returns the Catalog Table object for the TableName.
- * Adds the table to this analyzer's "missingTbls_" and throws an
AnalysisException if
- * the table has not yet been loaded in the local catalog cache.
- * Throws an AnalysisException if the table or the db does not exist in the
Catalog.
+ * Returns the Table with the given name from the 'loadedTables' map in the
global
+ * analysis state. Throws an AnalysisException if the table or the db does
not exist.
+ * Throws a TableLoadingException if the registered table failed to load.
* Always registers a privilege request for the table at the given privilege
level,
* regardless of the state of the table (i.e. whether it exists, is loaded,
etc.).
- * If addAccessEvent is true, adds an access event if the catalog access
succeeded.
+ * If addAccessEvent is true adds an access event for successfully loaded
tables.
*/
public Table getTable(TableName tableName, Privilege privilege, boolean
addAccessEvent)
throws AnalysisException, TableLoadingException {
Preconditions.checkNotNull(tableName);
Preconditions.checkNotNull(privilege);
- Table table = null;
- tableName = new TableName(getTargetDbName(tableName), tableName.getTbl());
-
+ tableName = getFqTableName(tableName);
if (privilege == Privilege.ANY) {
registerPrivReq(new PrivilegeRequestBuilder()
.any().onAnyColumn(tableName.getDb(),
tableName.getTbl()).toRequest());
@@ -2426,7 +2395,7 @@ public class Analyzer {
registerPrivReq(new PrivilegeRequestBuilder()
.allOf(privilege).onTable(tableName.getDb(),
tableName.getTbl()).toRequest());
}
- table = getTable(tableName.getDb(), tableName.getTbl());
+ Table table = getTable(tableName.getDb(), tableName.getTbl());
Preconditions.checkNotNull(table);
if (addAccessEvent) {
// Add an audit event for this access
@@ -2448,12 +2417,10 @@ public class Analyzer {
*/
public Table getTable(TableName tableName, Privilege privilege)
throws AnalysisException {
- // This may trigger a metadata load, in which case we want to return the
errors as
- // AnalysisExceptions.
try {
return getTable(tableName, privilege, true);
} catch (TableLoadingException e) {
- throw new AnalysisException(e.getMessage(), e);
+ throw new AnalysisException(e);
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/AuthorizationStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AuthorizationStmt.java
b/fe/src/main/java/org/apache/impala/analysis/AuthorizationStmt.java
index 9bfe2e6..c0186b3 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AuthorizationStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AuthorizationStmt.java
@@ -19,6 +19,7 @@ package org.apache.impala.analysis;
import org.apache.impala.authorization.User;
import org.apache.impala.common.AnalysisException;
+
import com.google.common.base.Strings;
/**
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
index 54daf7f..5df3dfa 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
@@ -187,6 +187,11 @@ public class ComputeStatsStmt extends StatementBase {
}
}
+ @Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(tableName_.toPath(), null));
+ }
+
/**
* Returns a stmt for COMPUTE STATS. The optional 'sampleParams' indicates
whether the
* stats should be computed with table sampling.
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
b/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
index 9ca14f4..c803910 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
@@ -18,8 +18,6 @@
package org.apache.impala.analysis;
import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.Db;
import org.apache.impala.common.AnalysisException;
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/CreateFunctionStmtBase.java
----------------------------------------------------------------------
diff --git
a/fe/src/main/java/org/apache/impala/analysis/CreateFunctionStmtBase.java
b/fe/src/main/java/org/apache/impala/analysis/CreateFunctionStmtBase.java
index 211b3de..5d8101f 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateFunctionStmtBase.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateFunctionStmtBase.java
@@ -22,7 +22,6 @@ import java.util.HashMap;
import java.util.List;
import org.apache.hadoop.fs.permission.FsAction;
-
import org.apache.impala.authorization.AuthorizeableFn;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.authorization.PrivilegeRequest;
@@ -33,6 +32,7 @@ import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TCreateFunctionParams;
import org.apache.impala.thrift.TFunctionBinaryType;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
----------------------------------------------------------------------
diff --git
a/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
b/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
index 701964a..4310871 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
@@ -80,6 +80,12 @@ public abstract class CreateOrAlterViewStmtBase extends
StatementBase {
this.viewDefStmt_ = viewDefStmt;
}
+ @Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(tableName_.toPath(), null));
+ viewDefStmt_.collectTableRefs(tblRefs);
+ }
+
/**
* Sets the originalViewDef and the expanded inlineViewDef based on
viewDefStmt.
* If columnDefs were given, checks that they do not contain duplicate
column names
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
----------------------------------------------------------------------
diff --git
a/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
b/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
index e050d06..c5c092d 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
@@ -92,6 +92,12 @@ public class CreateTableAsSelectStmt extends StatementBase {
public String toSql() { return ToSqlUtils.getCreateTableSql(this); }
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ createStmt_.collectTableRefs(tblRefs);
+ insertStmt_.collectTableRefs(tblRefs);
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
if (isAnalyzed()) return;
super.analyze(analyzer);
@@ -113,19 +119,14 @@ public class CreateTableAsSelectStmt extends
StatementBase {
// query portion of the insert statement. If this passes, analysis will be
run
// over the full INSERT statement. To avoid duplicate registrations of
table/colRefs,
// create a new root analyzer and clone the query statement for this
initial pass.
- Analyzer dummyRootAnalyzer = new Analyzer(analyzer.getCatalog(),
+ Analyzer dummyRootAnalyzer = new Analyzer(analyzer.getStmtTableCache(),
analyzer.getQueryCtx(), analyzer.getAuthzConfig());
QueryStmt tmpQueryStmt = insertStmt_.getQueryStmt().clone();
- try {
- Analyzer tmpAnalyzer = new Analyzer(dummyRootAnalyzer);
- tmpAnalyzer.setUseHiveColLabels(true);
- tmpQueryStmt.analyze(tmpAnalyzer);
- // Subqueries need to be rewritten by the StmtRewriter first.
- if (analyzer.containsSubquery()) return;
- } finally {
- // Record missing tables in the original analyzer.
- analyzer.getMissingTbls().addAll(dummyRootAnalyzer.getMissingTbls());
- }
+ Analyzer tmpAnalyzer = new Analyzer(dummyRootAnalyzer);
+ tmpAnalyzer.setUseHiveColLabels(true);
+ tmpQueryStmt.analyze(tmpAnalyzer);
+ // Subqueries need to be rewritten by the StmtRewriter first.
+ if (analyzer.containsSubquery()) return;
// Add the columns from the partition clause to the create statement.
if (partitionKeys_ != null) {
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
----------------------------------------------------------------------
diff --git
a/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
b/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
index 4091ef4..041ab9e 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
@@ -19,11 +19,7 @@ package org.apache.impala.analysis;
import java.util.List;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.permission.FsAction;
-
-import org.apache.impala.analysis.TableDef;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.Table;
@@ -34,6 +30,9 @@ import org.apache.impala.thrift.TCreateTableLikeParams;
import org.apache.impala.thrift.THdfsFileFormat;
import org.apache.impala.thrift.TTableName;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+
/**
* Represents a CREATE TABLE LIKE statement which creates a new table based on
* a copy of an existing table definition.
@@ -144,6 +143,12 @@ public class CreateTableLikeStmt extends StatementBase {
}
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(tableName_.toPath(), null));
+ tblRefs.add(new TableRef(srcTableName_.toPath(), null));
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
Preconditions.checkState(tableName_ != null && !tableName_.isEmpty());
Preconditions.checkState(srcTableName_ != null &&
!srcTableName_.isEmpty());
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
index 2e5425a..ee020df 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
@@ -182,6 +182,11 @@ public class CreateTableStmt extends StatementBase {
}
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(tableDef_.getTblName().toPath(), null));
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
super.analyze(analyzer);
owner_ = analyzer.getUser().getName();
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/DescribeDbStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/DescribeDbStmt.java
b/fe/src/main/java/org/apache/impala/analysis/DescribeDbStmt.java
index c6ffc9e..e7895b1 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DescribeDbStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DescribeDbStmt.java
@@ -21,6 +21,7 @@ import org.apache.impala.authorization.Privilege;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TDescribeDbParams;
import org.apache.impala.thrift.TDescribeOutputStyle;
+
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
b/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
index 6977f3b..90f8b07 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java
@@ -18,6 +18,7 @@
package org.apache.impala.analysis;
import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.impala.analysis.Path.PathType;
@@ -101,6 +102,11 @@ public class DescribeTableStmt extends StatementBase {
}
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(rawPath_, null));
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
try {
path_ = analyzer.resolvePath(rawPath_, PathType.ANY);
@@ -108,7 +114,6 @@ public class DescribeTableStmt extends StatementBase {
// Register privilege requests to prefer reporting an authorization
error over
// an analysis error. We should not accidentally reveal the
non-existence of a
// table/database if the user is not authorized.
- if (analyzer.hasMissingTbls()) throw ae;
if (rawPath_.size() > 1) {
analyzer.registerPrivReq(new PrivilegeRequestBuilder()
.onTable(rawPath_.get(0), rawPath_.get(1))
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java
b/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java
index e39071f..9ac0a11 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java
@@ -17,6 +17,8 @@
package org.apache.impala.analysis;
+import java.util.List;
+
import org.apache.impala.authorization.Privilege;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TDropStatsParams;
@@ -41,12 +43,12 @@ public class DropStatsStmt extends StatementBase {
* Constructor for building the DROP TABLE/VIEW statement
*/
public DropStatsStmt(TableName tableName) {
- this.tableName_ = tableName;
+ this.tableName_ = Preconditions.checkNotNull(tableName);
this.partitionSet_ = null;
}
public DropStatsStmt(TableName tableName, PartitionSet partitionSet) {
- this.tableName_ = tableName;
+ this.tableName_ = Preconditions.checkNotNull(tableName);;
this.partitionSet_ = partitionSet;
}
@@ -75,6 +77,11 @@ public class DropStatsStmt extends StatementBase {
return params;
}
+ @Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(tableName_.toPath(), null));
+ }
+
/**
* Checks that the given table exists and the user has privileges
* to drop stats on this table.
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java
----------------------------------------------------------------------
diff --git
a/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java
b/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java
index fc9db4d..72769d0 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java
@@ -17,6 +17,8 @@
package org.apache.impala.analysis;
+import java.util.List;
+
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.TableLoadingException;
@@ -26,6 +28,7 @@ import org.apache.impala.thrift.TAccessEvent;
import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TDropTableOrViewParams;
import org.apache.impala.thrift.TTableName;
+
import com.google.common.base.Preconditions;
/**
@@ -50,7 +53,7 @@ public class DropTableOrViewStmt extends StatementBase {
*/
public DropTableOrViewStmt(TableName tableName, boolean ifExists,
boolean dropTable, boolean purgeTable) {
- tableName_ = tableName;
+ tableName_ = Preconditions.checkNotNull(tableName);
ifExists_ = ifExists;
dropTable_ = dropTable;
purgeTable_ = purgeTable;
@@ -77,6 +80,11 @@ public class DropTableOrViewStmt extends StatementBase {
return params;
}
+ @Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(tableName_.toPath(), null));
+ }
+
/**
* 1. Checks that the user has privileges to DROP the given table/view
* 2. Checks that the database and table exists
@@ -108,8 +116,7 @@ public class DropTableOrViewStmt extends StatementBase {
analyzer.getFqTableName(tableName_).toString(),
TCatalogObjectType.TABLE,
Privilege.DROP.toString()));
} catch (AnalysisException e) {
- if (ifExists_ && analyzer.getMissingTbls().isEmpty()) return;
- throw e;
+ if (!ifExists_) throw e;
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/FromClause.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/FromClause.java
b/fe/src/main/java/org/apache/impala/analysis/FromClause.java
index 0526ca9..294c324 100644
--- a/fe/src/main/java/org/apache/impala/analysis/FromClause.java
+++ b/fe/src/main/java/org/apache/impala/analysis/FromClause.java
@@ -22,6 +22,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.impala.common.AnalysisException;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -51,35 +52,39 @@ public class FromClause implements ParseNode,
Iterable<TableRef> {
public void analyze(Analyzer analyzer) throws AnalysisException {
if (analyzed_) return;
- if (tableRefs_.isEmpty()) {
- analyzed_ = true;
- return;
- }
-
- // Start out with table refs to establish aliases.
TableRef leftTblRef = null; // the one to the left of tblRef
for (int i = 0; i < tableRefs_.size(); ++i) {
- // Resolve and replace non-InlineViewRef table refs with a BaseTableRef
or ViewRef.
TableRef tblRef = tableRefs_.get(i);
+ // Replace non-InlineViewRef table refs with a BaseTableRef or ViewRef.
tblRef = analyzer.resolveTableRef(tblRef);
tableRefs_.set(i, Preconditions.checkNotNull(tblRef));
tblRef.setLeftTblRef(leftTblRef);
- try {
- tblRef.analyze(analyzer);
- } catch (AnalysisException e) {
- // Only re-throw the exception if no tables are missing.
- if (analyzer.getMissingTbls().isEmpty()) throw e;
- }
+ tblRef.analyze(analyzer);
leftTblRef = tblRef;
}
+ analyzed_ = true;
+ }
- // All tableRefs have been analyzed, but at least one table is missing
metadata.
- if (!analyzer.getMissingTbls().isEmpty()) {
- throw new AnalysisException("Found missing tables. Aborting analysis.");
+ public void collectFromClauseTableRefs(List<TableRef> tblRefs) {
+ collectTableRefs(tblRefs, true);
+ }
+
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ collectTableRefs(tblRefs, false);
+ }
+
+ private void collectTableRefs(List<TableRef> tblRefs, boolean
fromClauseOnly) {
+ for (TableRef tblRef: tableRefs_) {
+ if (tblRef instanceof InlineViewRef) {
+ InlineViewRef inlineViewRef = (InlineViewRef) tblRef;
+ inlineViewRef.getViewStmt().collectTableRefs(tblRefs, fromClauseOnly);
+ } else {
+ tblRefs.add(tblRef);
+ }
}
- analyzed_ = true;
}
+ @Override
public FromClause clone() {
ArrayList<TableRef> clone = Lists.newArrayList();
for (TableRef tblRef: tableRefs_) clone.add(tblRef.clone());
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/GrantRevokePrivStmt.java
----------------------------------------------------------------------
diff --git
a/fe/src/main/java/org/apache/impala/analysis/GrantRevokePrivStmt.java
b/fe/src/main/java/org/apache/impala/analysis/GrantRevokePrivStmt.java
index d2c2040..9ded066 100644
--- a/fe/src/main/java/org/apache/impala/analysis/GrantRevokePrivStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/GrantRevokePrivStmt.java
@@ -23,6 +23,7 @@ import org.apache.impala.catalog.Role;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TGrantRevokePrivParams;
import org.apache.impala.thrift.TPrivilege;
+
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -79,6 +80,11 @@ public class GrantRevokePrivStmt extends AuthorizationStmt {
}
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ if (privilegeSpec_ != null) privilegeSpec_.collectTableRefs(tblRefs);
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
super.analyze(analyzer);
if (Strings.isNullOrEmpty(roleName_)) {
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
index 1d361d6..333dc54 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
@@ -20,7 +20,6 @@ package org.apache.impala.analysis;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import org.apache.impala.authorization.Privilege;
@@ -38,12 +37,9 @@ import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.planner.DataSink;
import org.apache.impala.planner.TableSink;
import org.apache.impala.rewrite.ExprRewriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -129,7 +125,7 @@ public class InsertStmt extends StatementBase {
// For Kudu tables, the primary keys are a leading subset of the cols, and
the partition
// cols can be any subset of the primary keys, meaning that this list will
be in
// ascending order from '0' to '# primary key cols - 1' but may leave out
some numbers.
- private List<Integer> partitionColPos_ = Lists.newArrayList();
+ private final List<Integer> partitionColPos_ = Lists.newArrayList();
// Indicates whether this insert stmt has a shuffle or noshuffle plan hint.
// Both flags may be false. Only one of them may be true, not both.
@@ -266,27 +262,17 @@ public class InsertStmt extends StatementBase {
public void analyze(Analyzer analyzer) throws AnalysisException {
if (isAnalyzed()) return;
super.analyze(analyzer);
- try {
- if (withClause_ != null) withClause_.analyze(analyzer);
- } catch (AnalysisException e) {
- // Ignore AnalysisExceptions if tables are missing to ensure the maximum
number
- // of missing tables can be collected before failing analyze().
- if (analyzer.getMissingTbls().isEmpty()) throw e;
- }
+ if (withClause_ != null) withClause_.analyze(analyzer);
List<Expr> selectListExprs = null;
if (!needsGeneratedQueryStatement_) {
- try {
- // Use a child analyzer for the query stmt to properly scope
WITH-clause
- // views and to ignore irrelevant ORDER BYs.
- Analyzer queryStmtAnalyzer = new Analyzer(analyzer);
- queryStmt_.analyze(queryStmtAnalyzer);
- // Use getResultExprs() and not getBaseTblResultExprs() here because
the final
- // substitution with TupleIsNullPredicate() wrapping happens in
planning.
- selectListExprs = Expr.cloneList(queryStmt_.getResultExprs());
- } catch (AnalysisException e) {
- if (analyzer.getMissingTbls().isEmpty()) throw e;
- }
+ // Use a child analyzer for the query stmt to properly scope WITH-clause
+ // views and to ignore irrelevant ORDER BYs.
+ Analyzer queryStmtAnalyzer = new Analyzer(analyzer);
+ queryStmt_.analyze(queryStmtAnalyzer);
+ // Use getResultExprs() and not getBaseTblResultExprs() here because the
final
+ // substitution with TupleIsNullPredicate() wrapping happens in planning.
+ selectListExprs = Expr.cloneList(queryStmt_.getResultExprs());
} else {
selectListExprs = Lists.newArrayList();
}
@@ -295,11 +281,6 @@ public class InsertStmt extends StatementBase {
// Also checks if the target table is missing.
analyzeTargetTable(analyzer);
- // Abort analysis if there are any missing tables beyond this point.
- if (!analyzer.getMissingTbls().isEmpty()) {
- throw new AnalysisException("Found missing tables. Aborting analysis.");
- }
-
boolean isHBaseTable = (table_ instanceof HBaseTable);
int numClusteringCols = isHBaseTable ? 0 : table_.getNumClusteringCols();
@@ -947,4 +928,15 @@ public class InsertStmt extends StatementBase {
}
return strBuilder.toString();
}
+
+ @Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ if (withClause_ != null) {
+ for (View v: withClause_.getViews()) {
+ v.getQueryStmt().collectTableRefs(tblRefs);
+ }
+ }
+ tblRefs.add(new TableRef(targetTableName_.toPath(), null));
+ if (queryStmt_ != null) queryStmt_.collectTableRefs(tblRefs);
+ }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/LimitElement.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/LimitElement.java
b/fe/src/main/java/org/apache/impala/analysis/LimitElement.java
index d357b07..ab5dac1 100644
--- a/fe/src/main/java/org/apache/impala/analysis/LimitElement.java
+++ b/fe/src/main/java/org/apache/impala/analysis/LimitElement.java
@@ -21,6 +21,7 @@ import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.InternalException;
import org.apache.impala.service.FeSupport;
import org.apache.impala.thrift.TColumnValue;
+
import com.google.common.base.Preconditions;
/**
@@ -118,9 +119,9 @@ class LimitElement {
*/
private static long evalIntegerExpr(Analyzer analyzer, Expr expr, String
name)
throws AnalysisException {
- // Check for slotrefs before analysis so we can provide a more helpful
message than
- // "Could not resolve column/field reference".
- if (expr.contains(SlotRef.class)) {
+ // Check for slotrefs and subqueries before analysis so we can provide a
more
+ // helpful error message.
+ if (expr.contains(SlotRef.class) || expr.contains(Subquery.class)) {
throw new AnalysisException(name + " expression must be a constant
expression: " +
expr.toSql());
}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
b/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
index ec0244d..114862e 100644
--- a/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
@@ -19,13 +19,14 @@ package org.apache.impala.analysis;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.List;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.AdlFileSystem;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.adl.AdlFileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.HdfsFileFormat;
@@ -99,6 +100,11 @@ public class LoadDataStmt extends StatementBase {
}
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(tableName_.toPath(), null));
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
dbName_ = analyzer.getTargetDbName(tableName_);
Table table = analyzer.getTable(tableName_, Privilege.INSERT);
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
index 5bac75c..32b4a1d 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
@@ -33,7 +33,6 @@ import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.Pair;
import org.apache.impala.planner.DataSink;
import org.apache.impala.rewrite.ExprRewriter;
-import org.slf4j.LoggerFactory;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
@@ -98,6 +97,20 @@ public abstract class ModifyStmt extends StatementBase {
wherePredicate_ = wherePredicate;
}
+ @Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(targetTablePath_, null));
+ fromClause_.collectTableRefs(tblRefs);
+ if (wherePredicate_ != null) {
+ // Collect TableRefs in WHERE-clause subqueries.
+ List<Subquery> subqueries = Lists.newArrayList();
+ wherePredicate_.collect(Subquery.class, subqueries);
+ for (Subquery sq : subqueries) {
+ sq.getStatement().collectTableRefs(tblRefs);
+ }
+ }
+ }
+
/**
* The analysis of the ModifyStmt proceeds as follows: First, the FROM
clause is
* analyzed and the targetTablePath is verified to be a valid alias into the
FROM
@@ -232,12 +245,18 @@ public abstract class ModifyStmt extends StatementBase {
// Assignments are only used in the context of updates.
for (Pair<SlotRef, Expr> valueAssignment : assignments_) {
- Expr rhsExpr = valueAssignment.second;
- rhsExpr.analyze(analyzer);
-
SlotRef lhsSlotRef = valueAssignment.first;
lhsSlotRef.analyze(analyzer);
+ Expr rhsExpr = valueAssignment.second;
+ // No subqueries for rhs expression
+ if (rhsExpr.contains(Subquery.class)) {
+ throw new AnalysisException(
+ format("Subqueries are not supported as update expressions for
column '%s'",
+ lhsSlotRef.toSql()));
+ }
+ rhsExpr.analyze(analyzer);
+
// Correct target table
if (!lhsSlotRef.isBoundByTupleIds(targetTableRef_.getId().asList())) {
throw new AnalysisException(
@@ -246,13 +265,6 @@ public abstract class ModifyStmt extends StatementBase {
rhsExpr.toSql(),
targetTableRef_.getDesc().getTable().getFullName()));
}
- // No subqueries for rhs expression
- if (rhsExpr.contains(Subquery.class)) {
- throw new AnalysisException(
- format("Subqueries are not supported as update expressions for
column '%s'",
- lhsSlotRef.toSql()));
- }
-
Column c = lhsSlotRef.getResolvedPath().destColumn();
// TODO(Kudu) Add test for this code-path when Kudu supports nested types
if (c == null) {
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/Path.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/Path.java
b/fe/src/main/java/org/apache/impala/analysis/Path.java
index 32ba3d5..9eb57a0 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Path.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Path.java
@@ -27,6 +27,7 @@ import org.apache.impala.catalog.StructField;
import org.apache.impala.catalog.StructType;
import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.Type;
+
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -270,6 +271,28 @@ public class Path {
}
}
+ /**
+ * Returns a list of table names that might be referenced by the given path.
+ * The path must be non-empty.
+ *
+ * Examples: path -> result
+ * a -> [<sessionDb>.a]
+ * a.b -> [<sessionDb>.a, a.b]
+ * a.b.c -> [<sessionDb>.a, a.b]
+ * a.b.c... -> [<sessionDb>.a, a.b]
+ */
+ public static List<TableName> getCandidateTables(List<String> path, String
sessionDb) {
+ Preconditions.checkArgument(path != null && !path.isEmpty());
+ List<TableName> result = Lists.newArrayList();
+ int end = Math.min(2, path.size());
+ for (int tblNameIdx = 0; tblNameIdx < end; ++tblNameIdx) {
+ String dbName = (tblNameIdx == 0) ? sessionDb : path.get(0);
+ String tblName = path.get(tblNameIdx);
+ result.add(new TableName(dbName, tblName));
+ }
+ return result;
+ }
+
public Table getRootTable() { return rootTable_; }
public TupleDescriptor getRootDesc() { return rootDesc_; }
public boolean isRootedAtTable() { return rootTable_ != null; }
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
b/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
index 9e8731a..a21af93 100644
--- a/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
+++ b/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
@@ -21,7 +21,6 @@ import java.util.List;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.DataSourceTable;
-import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.RolePrivilege;
import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.TableLoadingException;
@@ -30,6 +29,7 @@ import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TPrivilege;
import org.apache.impala.thrift.TPrivilegeLevel;
import org.apache.impala.thrift.TPrivilegeScope;
+
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -171,6 +171,10 @@ public class PrivilegeSpec implements ParseNode {
return sb.toString();
}
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ if (tableName_ != null) tblRefs.add(new TableRef(tableName_.toPath(),
null));
+ }
+
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
String configServerName = analyzer.getAuthzConfig().getServerName();
@@ -277,7 +281,6 @@ public class PrivilegeSpec implements ParseNode {
} catch (TableLoadingException e) {
throw new AnalysisException(e.getMessage(), e);
} catch (AnalysisException e) {
- if (analyzer.hasMissingTbls()) throw e;
throw new AnalysisException(String.format("Error setting privileges for
" +
"table '%s'. Verify that the table exists and that you have
permissions " +
"to issue a GRANT/REVOKE statement.", tableName_.toString()));
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/QueryStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/QueryStmt.java
b/fe/src/main/java/org/apache/impala/analysis/QueryStmt.java
index 178e80c..1ebee10 100644
--- a/fe/src/main/java/org/apache/impala/analysis/QueryStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/QueryStmt.java
@@ -23,6 +23,7 @@ import java.util.ListIterator;
import java.util.Set;
import org.apache.impala.catalog.Type;
+import org.apache.impala.catalog.View;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.planner.DataSink;
import org.apache.impala.planner.PlanRootSink;
@@ -95,6 +96,32 @@ public abstract class QueryStmt extends StatementBase {
ambiguousAliasList_ = Lists.newArrayList();
}
+ /**
+ * Returns all table references in the FROM clause of this statement and all
statements
+ * nested within FROM clauses.
+ */
+ public void collectFromClauseTableRefs(List<TableRef> tblRefs) {
+ collectTableRefs(tblRefs, true);
+ }
+
+ @Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ collectTableRefs(tblRefs, false);
+ }
+
+ /**
+ * Helper for collectFromClauseTableRefs() and collectTableRefs().
+ * If 'fromClauseOnly' is true only collects table references in the FROM
clause,
+ * otherwise all table references.
+ */
+ protected void collectTableRefs(List<TableRef> tblRefs, boolean
fromClauseOnly) {
+ if (!fromClauseOnly && withClause_ != null) {
+ for (View v: withClause_.getViews()) {
+ v.getQueryStmt().collectTableRefs(tblRefs, fromClauseOnly);
+ }
+ }
+ }
+
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
if (isAnalyzed()) return;
@@ -132,7 +159,7 @@ public abstract class QueryStmt extends StatementBase {
Set<TupleId> tblRefIds = Sets.newHashSet();
List<TableRef> tblRefs = Lists.newArrayList();
- collectTableRefs(tblRefs);
+ collectTableRefs(tblRefs, true);
for (TableRef tblRef: tblRefs) {
if (absoluteRef == null && !tblRef.isRelative()) absoluteRef = tblRef;
if (tblRef.isCorrelated()) {
@@ -339,12 +366,6 @@ public abstract class QueryStmt extends StatementBase {
*/
public abstract void getMaterializedTupleIds(ArrayList<TupleId> tupleIdList);
- /**
- * Returns all physical (non-inline-view) TableRefs of this statement and
the nested
- * statements of inline views. The returned TableRefs are in depth-first
order.
- */
- public abstract void collectTableRefs(List<TableRef> tblRefs);
-
@Override
public List<Expr> getResultExprs() { return resultExprs_; }
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
index ac7ca2e..1fd1e7c 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
@@ -17,6 +17,8 @@
package org.apache.impala.analysis;
+import java.util.List;
+
import org.apache.impala.authorization.Privilege;
import org.apache.impala.authorization.PrivilegeRequest;
import org.apache.impala.authorization.PrivilegeRequestBuilder;
@@ -78,6 +80,11 @@ public class ResetMetadataStmt extends StatementBase {
public TableName getTableName() { return tableName_; }
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ if (tableName_ != null) tblRefs.add(new TableRef(tableName_.toPath(),
null));
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
if (tableName_ != null) {
String dbName = analyzer.getTargetDbName(tableName_);
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
index f9d08b7..a3bac51 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
@@ -151,6 +151,7 @@ public class SelectStmt extends QueryStmt {
if (isAnalyzed()) return;
super.analyze(analyzer);
+ // Start out with table refs to establish aliases.
fromClause_.analyze(analyzer);
// Generate !empty() predicates to filter out empty collections.
@@ -541,12 +542,12 @@ public class SelectStmt extends QueryStmt {
// Analyze the HAVING clause first so we can check if it contains
aggregates.
// We need to analyze/register it even if we are not computing aggregates.
if (havingClause_ != null) {
- havingPred_ = substituteOrdinalOrAlias(havingClause_, "HAVING",
analyzer);
// can't contain subqueries
- if (havingPred_.contains(Predicates.instanceOf(Subquery.class))) {
+ if (havingClause_.contains(Predicates.instanceOf(Subquery.class))) {
throw new AnalysisException(
"Subqueries are not supported in the HAVING clause.");
}
+ havingPred_ = substituteOrdinalOrAlias(havingClause_, "HAVING",
analyzer);
// can't contain analytic exprs
Expr analyticExpr = havingPred_.findFirstOf(AnalyticExpr.class);
if (analyticExpr != null) {
@@ -1041,13 +1042,19 @@ public class SelectStmt extends QueryStmt {
}
@Override
- public void collectTableRefs(List<TableRef> tblRefs) {
- for (TableRef tblRef: fromClause_) {
- if (tblRef instanceof InlineViewRef) {
- InlineViewRef inlineViewRef = (InlineViewRef) tblRef;
- inlineViewRef.getViewStmt().collectTableRefs(tblRefs);
- } else {
- tblRefs.add(tblRef);
+ protected void collectTableRefs(List<TableRef> tblRefs, boolean
fromClauseOnly) {
+ super.collectTableRefs(tblRefs, fromClauseOnly);
+ if (fromClauseOnly) {
+ fromClause_.collectFromClauseTableRefs(tblRefs);
+ } else {
+ fromClause_.collectTableRefs(tblRefs);
+ }
+ if (!fromClauseOnly && whereClause_ != null) {
+ // Collect TableRefs in WHERE-clause subqueries.
+ List<Subquery> subqueries = Lists.newArrayList();
+ whereClause_.collect(Subquery.class, subqueries);
+ for (Subquery sq: subqueries) {
+ sq.getStatement().collectTableRefs(tblRefs, fromClauseOnly);
}
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/SetStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/SetStmt.java
b/fe/src/main/java/org/apache/impala/analysis/SetStmt.java
index a83efda..5a48b93 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SetStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SetStmt.java
@@ -18,6 +18,7 @@
package org.apache.impala.analysis;
import org.apache.impala.thrift.TSetQueryOptionRequest;
+
import com.google.common.base.Preconditions;
/**
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/ShowCreateFunctionStmt.java
----------------------------------------------------------------------
diff --git
a/fe/src/main/java/org/apache/impala/analysis/ShowCreateFunctionStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ShowCreateFunctionStmt.java
index 37e32ae..414bbd7 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ShowCreateFunctionStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ShowCreateFunctionStmt.java
@@ -19,13 +19,13 @@ package org.apache.impala.analysis;
import java.util.List;
-import org.apache.impala.analysis.FunctionName;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.Db;
import org.apache.impala.catalog.Function;
import org.apache.impala.common.AnalysisException;
-import org.apache.impala.thrift.TGetFunctionsParams;
import org.apache.impala.thrift.TFunctionCategory;
+import org.apache.impala.thrift.TGetFunctionsParams;
+
import com.google.common.base.Preconditions;
/**
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/ShowCreateTableStmt.java
----------------------------------------------------------------------
diff --git
a/fe/src/main/java/org/apache/impala/analysis/ShowCreateTableStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ShowCreateTableStmt.java
index d2b57e8..216562b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ShowCreateTableStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ShowCreateTableStmt.java
@@ -17,12 +17,15 @@
package org.apache.impala.analysis;
+import java.util.List;
+
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.View;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TTableName;
+
import com.google.common.base.Preconditions;
/**
@@ -36,7 +39,7 @@ public class ShowCreateTableStmt extends StatementBase {
private TableName tableName_;
// The object type keyword used, e.g. TABLE or VIEW, needed to output
matching SQL.
- private TCatalogObjectType objectType_;
+ private final TCatalogObjectType objectType_;
public ShowCreateTableStmt(TableName table, TCatalogObjectType objectType) {
Preconditions.checkNotNull(table);
@@ -50,6 +53,11 @@ public class ShowCreateTableStmt extends StatementBase {
}
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(tableName_.toPath(), null));
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
tableName_ = analyzer.getFqTableName(tableName_);
Table table = analyzer.getTable(tableName_, Privilege.VIEW_METADATA);
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
index 42839dd..ee749d0 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
@@ -17,6 +17,8 @@
package org.apache.impala.analysis;
+import java.util.List;
+
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.Table;
@@ -43,8 +45,8 @@ public class ShowFilesStmt extends StatementBase {
protected Table table_;
public ShowFilesStmt(TableName tableName, PartitionSet partitionSet) {
- this.tableName_ = tableName;
- this.partitionSet_ = partitionSet;
+ tableName_ = Preconditions.checkNotNull(tableName);
+ partitionSet_ = partitionSet;
}
@Override
@@ -56,6 +58,11 @@ public class ShowFilesStmt extends StatementBase {
}
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(tableName_.toPath(), null));
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
// Resolve and analyze table ref to register privilege and audit events
// and to allow us to evaluate partition predicates.
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/ShowFunctionsStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ShowFunctionsStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ShowFunctionsStmt.java
index dc6a461..296fafd 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ShowFunctionsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ShowFunctionsStmt.java
@@ -21,6 +21,7 @@ import org.apache.impala.authorization.Privilege;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TFunctionCategory;
import org.apache.impala.thrift.TShowFunctionsParams;
+
import com.google.common.base.Preconditions;
/**
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/ShowGrantRoleStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ShowGrantRoleStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ShowGrantRoleStmt.java
index e59f168..82c6ed0 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ShowGrantRoleStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ShowGrantRoleStmt.java
@@ -17,10 +17,13 @@
package org.apache.impala.analysis;
+import java.util.List;
+
import org.apache.impala.catalog.Role;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.InternalException;
import org.apache.impala.thrift.TShowGrantRoleParams;
+
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -60,6 +63,11 @@ public class ShowGrantRoleStmt extends AuthorizationStmt {
}
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ if (privilegeSpec_ != null) privilegeSpec_.collectTableRefs(tblRefs);
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
super.analyze(analyzer);
if (Strings.isNullOrEmpty(roleName_)) {
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/ShowStatsStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ShowStatsStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ShowStatsStmt.java
index 8801b0a..91dfe75 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ShowStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ShowStatsStmt.java
@@ -17,6 +17,8 @@
package org.apache.impala.analysis;
+import java.util.List;
+
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.KuduTable;
@@ -25,6 +27,7 @@ import org.apache.impala.catalog.View;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TShowStatsOp;
import org.apache.impala.thrift.TShowStatsParams;
+
import com.google.common.base.Preconditions;
/**
@@ -39,8 +42,8 @@ public class ShowStatsStmt extends StatementBase {
protected Table table_;
public ShowStatsStmt(TableName tableName, TShowStatsOp op) {
- this.op_ = op;
- this.tableName_ = tableName;
+ op_ = Preconditions.checkNotNull(op);
+ tableName_ = Preconditions.checkNotNull(tableName);
}
@Override
@@ -64,6 +67,11 @@ public class ShowStatsStmt extends StatementBase {
}
@Override
+ public void collectTableRefs(List<TableRef> tblRefs) {
+ tblRefs.add(new TableRef(tableName_.toPath(), null));
+ }
+
+ @Override
public void analyze(Analyzer analyzer) throws AnalysisException {
table_ = analyzer.getTable(tableName_, Privilege.VIEW_METADATA);
Preconditions.checkNotNull(table_);
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/main/java/org/apache/impala/analysis/ShowTablesStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ShowTablesStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ShowTablesStmt.java
index a6a23c9..bf2612b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ShowTablesStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ShowTablesStmt.java
@@ -20,6 +20,7 @@ package org.apache.impala.analysis;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TShowTablesParams;
+
import com.google.common.base.Preconditions;
/**
