Repository: incubator-impala Updated Branches: refs/heads/master 9ce691af2 -> 545eab6d6
IMPALA-2810: Remove column stats restoration when altering table This patch removes most code introduced in IMPALA-1711, which triggers an error message and causes catalog inconsistency when a partitioned table is moved to a different database. IMPALA-1711 is a workaround for HIVE-9720, in which column stats is not properly updated when a table is moved across databases. Hive-9720 has been resolved in Hive 1.2.0 so the workaround is no longer needed. A test case moving a partitioned table to a different database is added to alter-table.test. Change-Id: I0ca7063ca1aa9faceed9568d22740d91b6dc20d3 Reviewed-on: http://gerrit.cloudera.org:8080/7857 Reviewed-by: Alex Behm <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f8e7c31b Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f8e7c31b Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f8e7c31b Branch: refs/heads/master Commit: f8e7c31b2c681da91c02011fe2207c30dec3d6c5 Parents: 9ce691a Author: Tianyi Wang <[email protected]> Authored: Fri Aug 25 17:41:13 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Wed Sep 6 03:05:38 2017 +0000 ---------------------------------------------------------------------- .../impala/service/CatalogOpExecutor.java | 34 +---------- .../queries/QueryTest/alter-table.test | 61 +++++++++++++++++++- 2 files changed, 59 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f8e7c31b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java index 675ced5..e791de1 100644 --- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java +++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java @@ -2173,8 +2173,7 @@ public class CatalogOpExecutor { } /** - * Renames an existing table or view. Saves, drops and restores the column stats for - * tables renamed across databases to work around HIVE-9720/IMPALA-1711. + * Renames an existing table or view. * After renaming the table/view, its metadata is marked as invalid and will be * reloaded on the next access. */ @@ -2188,38 +2187,7 @@ public class CatalogOpExecutor { msTbl.setDbName(newTableName.getDb()); msTbl.setTableName(newTableName.getTbl()); try (MetaStoreClient msClient = catalog_.getMetaStoreClient()) { - // Workaround for HIVE-9720/IMPALA-1711: When renaming a table with column - // stats across databases, we save, drop and restore the column stats because - // the HMS does not properly move them to the new table via alteration. - ColumnStatistics hmsColStats = null; - if (!msTbl.getTableType().equalsIgnoreCase(TableType.VIRTUAL_VIEW.toString()) - && !tableName.getDb().equalsIgnoreCase(newTableName.getDb())) { - Map<String, TColumnStats> colStats = Maps.newHashMap(); - for (Column c: oldTbl.getColumns()) { - colStats.put(c.getName(), c.getStats().toThrift()); - } - hmsColStats = createHiveColStats(colStats, oldTbl); - // Set the new db/table. - hmsColStats.setStatsDesc(new ColumnStatisticsDesc(true, newTableName.getDb(), - newTableName.getTbl())); - - LOG.trace(String.format("Dropping column stats for table %s being " + - "renamed to %s to workaround HIVE-9720.", - tableName.toString(), newTableName.toString())); - // Delete all column stats of the original table from the HMS. - msClient.getHiveClient().deleteTableColumnStatistics( - tableName.getDb(), tableName.getTbl(), null); - } - - // Perform the table rename in any case. msClient.getHiveClient().alter_table(tableName.getDb(), tableName.getTbl(), msTbl); - - if (hmsColStats != null) { - LOG.trace(String.format("Restoring column stats for table %s being " + - "renamed to %s to workaround HIVE-9720.", - tableName.toString(), newTableName.toString())); - msClient.getHiveClient().updateTableColumnStatistics(hmsColStats); - } } catch (TException e) { throw new ImpalaRuntimeException( String.format(HMS_RPC_ERROR_FORMAT_STR, "alter_table"), e); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f8e7c31b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test index 719f4f0..e2bcf4e 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test +++ b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test @@ -771,14 +771,15 @@ invalidate metadata $DATABASE2.mv2 ---- RESULTS ==== ---- QUERY -show tables in $DATABASE like '%mv%' +show tables in $DATABASE like '*mv*' ---- RESULTS ---- TYPES STRING ==== ---- QUERY -show tables in $DATABASE2 like '%mv%' +show tables in $DATABASE2 like '*mv*' ---- RESULTS +'mv2' ---- TYPES STRING ==== @@ -802,7 +803,61 @@ drop table $DATABASE2.mv2 ---- RESULTS ==== ---- QUERY -show tables in $DATABASE2 like '%mv%' +show tables in $DATABASE2 like '*mv*' +---- RESULTS +---- TYPES +STRING +==== +---- QUERY +# Tests that renaming a partitioned table with column stats across databases +# succeeds and preserves table and column stats, and allows the renamed table +# to be dropped (IMPALA-2810). +create table $DATABASE.mv (x int) partitioned by (y string); +insert into $DATABASE.mv partition(y='a') values(1); +insert into $DATABASE.mv partition(y='b') values(2); +insert into $DATABASE.mv partition(y=NULL) values(NULL); +compute stats $DATABASE.mv; +alter table $DATABASE.mv rename to $DATABASE2.mv2; +invalidate metadata $DATABASE2.mv2 +---- RESULTS +==== +---- QUERY +show tables in $DATABASE like '*mv*' +---- RESULTS +---- TYPES +STRING +==== +---- QUERY +show tables in $DATABASE2 like '*mv*' +---- RESULTS +'mv2' +---- TYPES +STRING +==== +---- QUERY +show table stats $DATABASE2.mv2 +---- RESULTS: VERIFY_IS_EQUAL_SORTED +'NULL',1,1,'3B','NOT CACHED','NOT CACHED','TEXT','false',regex:.* +'a',1,1,'2B','NOT CACHED','NOT CACHED','TEXT','false',regex:.* +'b',1,1,'2B','NOT CACHED','NOT CACHED','TEXT','false',regex:.* +'Total',3,3,'7B','0B','','','','' +---- TYPES +STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING +==== +---- QUERY +show column stats $DATABASE2.mv2 +---- RESULTS: VERIFY_IS_EQUAL_SORTED +'x','INT',2,-1,4,4 +'y','STRING',3,1,-1,-1 +---- TYPES +STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE +==== +---- QUERY +drop table $DATABASE2.mv2 +---- RESULTS +==== +---- QUERY +show tables in $DATABASE2 like '*mv*' ---- RESULTS ---- TYPES STRING
