Repository: incubator-impala Updated Branches: refs/heads/master 537ae013e -> 495325440
IMPALA-5954: Set DO_NOT_UPDATE_STATS in alterTable() RPCs to HMS. This improves an existing workaround for HIVE-12730 to avoid having the HMS wipe stats as a side-effect of an alterTable() RPC. The new workaround uses DO_NOT_UPDATE_STATS which is the recommended way of telling the HMS not to recompute/reset statistics on its own. Testing: - core/hdfs run passed Change-Id: I05c63315579d66efeaa4c65105372ae68820ab2f Reviewed-on: http://gerrit.cloudera.org:8080/8112 Reviewed-by: Alex Behm <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/eb276d48 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/eb276d48 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/eb276d48 Branch: refs/heads/master Commit: eb276d48e4fc651d910ccb3af78a235eac190905 Parents: 537ae01 Author: Alex Behm <[email protected]> Authored: Mon Sep 18 18:41:10 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Fri Sep 22 00:53:49 2017 +0000 ---------------------------------------------------------------------- .../org/apache/impala/service/CatalogOpExecutor.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/eb276d48/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java index e791de1..cf1e677 100644 --- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java +++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java @@ -797,6 +797,7 @@ public class CatalogOpExecutor { } PartitionStatsUtil.partStatsToParameters(partitionStats, partition); partition.putToParameters(StatsSetupConst.ROW_COUNT, String.valueOf(numRows)); + // HMS requires this param for stats changes to take effect. partition.putToParameters(MetastoreShim.statsGeneratedViaStatsTaskParam()); ++numTargetedPartitions; modifiedParts.add(partition); @@ -819,6 +820,7 @@ public class CatalogOpExecutor { msTbl.putToParameters(StatsSetupConst.RAW_DATA_SIZE, String.valueOf(params.getTable_stats().total_file_bytes)); } + // HMS requires this param for stats changes to take effect. Pair<String, String> statsTaskParam = MetastoreShim.statsGeneratedViaStatsTaskParam(); msTbl.putToParameters(statsTaskParam.first, statsTaskParam.second); return numTargetedPartitions; @@ -2729,7 +2731,8 @@ public class CatalogOpExecutor { } } - /** Applies an ALTER TABLE command to the metastore table. + /** + * Applies an ALTER TABLE command to the metastore table. * Note: The metastore interface is not very safe because it only accepts * an entire metastore.api.Table object rather than a delta of what to change. This * means an external modification to the table could be overwritten by an ALTER TABLE @@ -2743,11 +2746,11 @@ public class CatalogOpExecutor { try (MetaStoreClient msClient = catalog_.getMetaStoreClient()) { lastDdlTime = calculateDdlTime(msTbl); msTbl.putToParameters("transient_lastDdlTime", Long.toString(lastDdlTime)); - // TODO: Remove this workaround for HIVE-15653 to preserve table stats - // during table alterations. - Pair<String, String> statsTaskParam = - MetastoreShim.statsGeneratedViaStatsTaskParam(); - msTbl.putToParameters(statsTaskParam.first, statsTaskParam.second); + // Avoid computing/setting stats on the HMS side because that may reset the + // 'numRows' table property (see HIVE-15653). The DO_NOT_UPDATE_STATS flag + // tells the HMS not to recompute/reset any statistics on its own. Any + // stats-related alterations passed in the RPC will still be applied. + msTbl.putToParameters(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); msClient.getHiveClient().alter_table( msTbl.getDbName(), msTbl.getTableName(), msTbl); } catch (TException e) {
