Repository: impala Updated Branches: refs/heads/master 1765a44da -> ad91e0b04
IMPALA-6567: ResetMetadataStmt analysis should not load tables. This fixes a regression introduced by IMPALA-5152 where invalidate metadata <tbl> and refresh <tbl> accidentally required the target table to be loaded during analysis, ultimately leading to a double load in some situations (load during analysis, then another load during execution). Since the purpose of these statements is to reload metadata it does not make sense to require a table load during analysis - that load happens during execution. Note that REFRESH <tbl> PARTITION (<partition>) still requires the containing table to be loaded. This was the behavior before IMPALA-5152 and this patch does not attempt to improve that. Testing: - added new unit test - ran FE tests locally - validated the desired behavior by inspecting logs and the timeine from invalidate/refresh statements Change-Id: I7033781ebf27ea53cfd26ff0e4f74d4f242bd1dc Reviewed-on: http://gerrit.cloudera.org:8080/9418 Tested-by: Impala Public Jenkins Reviewed-by: Alex Behm <alex.b...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ad91e0b0 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ad91e0b0 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ad91e0b0 Branch: refs/heads/master Commit: ad91e0b04cedb84b5b08c810de4ab1a5555ef036 Parents: 1765a44 Author: Alex Behm <alex.b...@cloudera.com> Authored: Thu Feb 22 21:07:27 2018 -0800 Committer: Alex Behm <alex.b...@cloudera.com> Committed: Fri Feb 23 21:16:27 2018 +0000 ---------------------------------------------------------------------- .../impala/analysis/ResetMetadataStmt.java | 5 ++++- .../impala/analysis/StmtMetadataLoaderTest.java | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/ad91e0b0/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java index 1fd1e7c..e070d51 100644 --- a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java +++ b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java @@ -81,7 +81,10 @@ public class ResetMetadataStmt extends StatementBase { @Override public void collectTableRefs(List<TableRef> tblRefs) { - if (tableName_ != null) tblRefs.add(new TableRef(tableName_.toPath(), null)); + // Only need table metadata for REFRESH <tbl> PARTITION (<partition>) + if (tableName_ != null && partitionSpec_ != null) { + tblRefs.add(new TableRef(tableName_.toPath(), null)); + } } @Override http://git-wip-us.apache.org/repos/asf/impala/blob/ad91e0b0/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java b/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java index f2c8faa..39416f8 100644 --- a/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java @@ -47,6 +47,13 @@ public class StmtMetadataLoaderTest { validateCached(stmt, fe, expectedDbs, expectedTables); } + private void testNoLoad(String stmtStr) throws ImpalaException { + ImpaladTestCatalog catalog = new ImpaladTestCatalog(); + Frontend fe = new Frontend(AuthorizationConfig.createAuthDisabledConfig(), catalog); + StatementBase stmt = fe.parse(stmtStr); + validateCached(stmt, fe, new String[]{}, new String[]{}); + } + private void validateDbs(StmtTableCache stmtTableCache, String[] expectedDbs) { String[] actualDbs = new String[stmtTableCache.dbs.size()]; actualDbs = stmtTableCache.dbs.toArray(actualDbs); @@ -177,4 +184,17 @@ public class StmtMetadataLoaderTest { new String[] {"functional.view_view", "functional.alltypes_view", "functional.alltypes"}); } + + @Test + public void testResetMetadataStmts() throws ImpalaException { + // These stmts should not request any table loads. + testNoLoad("invalidate metadata"); + testNoLoad("invalidate metadata functional.alltypes"); + testNoLoad("refresh functional.alltypes"); + testNoLoad("refresh functions functional"); + + // This stmt requires the table to be loaded. + testLoadTables("refresh functional.alltypes partition (year=2009, month=1)", 1, 1, + new String[] {"default", "functional"}, new String[] {"functional.alltypes"}); + } }