HIVE-18138: Fix columnstats problem in case schema evolution (Zoltan Haindrich, reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c2fc0fb8 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c2fc0fb8 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c2fc0fb8 Branch: refs/heads/master Commit: c2fc0fb88b794f07d9d4b1cd5f4ef3e1b8737911 Parents: a1f54df Author: Zoltan Haindrich <k...@rxd.hu> Authored: Wed Dec 6 09:35:48 2017 +0100 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Wed Dec 6 09:47:55 2017 +0100 ---------------------------------------------------------------------- .../hadoop/hive/metastore/HiveAlterHandler.java | 14 +++++++++++--- .../org/apache/hadoop/hive/metastore/ObjectStore.java | 8 +++++--- 2 files changed, 16 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c2fc0fb8/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index b445723..13967d5 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -300,21 +300,29 @@ public class HiveAlterHandler implements AlterHandler { MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true, environmentContext); } - if (cascade && isPartitionedTable) { + if (isPartitionedTable) { //Currently only column related changes can be cascaded in alter table if(!MetaStoreUtils.areSameColumns(oldt.getSd().getCols(), newt.getSd().getCols())) { parts = msdb.getPartitions(dbname, name, -1); for (Partition part : parts) { + Partition oldPart = new Partition(part); List<FieldSchema> oldCols = part.getSd().getCols(); part.getSd().setCols(newt.getSd().getCols()); ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, dbname, name, part.getValues(), oldCols, oldt, part, null); assert(colStats == null); - msdb.alterPartition(dbname, name, part.getValues(), part); + if (cascade) { + msdb.alterPartition(dbname, name, part.getValues(), part); + } else { + // update changed properties (stats) + oldPart.setParameters(part.getParameters()); + msdb.alterPartition(dbname, name, part.getValues(), oldPart); + } } msdb.alterTable(dbname, name, newt); } else { - LOG.warn("Alter table does not cascade changes to its partitions."); + LOG.warn("Alter table not cascaded to partitions."); + alterTableUpdateTableColumnStats(msdb, oldt, newt); } } else { alterTableUpdateTableColumnStats(msdb, oldt, newt); http://git-wip-us.apache.org/repos/asf/hive/blob/c2fc0fb8/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 0818704..2e80c9d 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -3500,7 +3500,7 @@ public class ObjectStore implements RawStore, Configurable { LOG.debug("filter specified is {}, JDOQL filter is {}", filter, queryFilterString); if (LOG.isDebugEnabled()) { for (Entry<String, Object> entry : params.entrySet()) { - LOG.debug("key: {} value: {} class: {}", entry.getKey(), entry.getValue(), + LOG.debug("key: {} value: {} class: {}", entry.getKey(), entry.getValue(), entry.getValue().getClass().getName()); } } @@ -7665,7 +7665,7 @@ public class ObjectStore implements RawStore, Configurable { private List<MTableColumnStatistics> getMTableColumnStatistics(Table table, List<String> colNames, QueryWrapper queryWrapper) throws MetaException { if (colNames == null || colNames.isEmpty()) { - return null; + return Collections.emptyList(); } boolean committed = false; @@ -7750,7 +7750,9 @@ public class ObjectStore implements RawStore, Configurable { try { List<MTableColumnStatistics> mStats = getMTableColumnStatistics(getTable(), colNames, queryWrapper); - if (mStats.isEmpty()) return null; + if (mStats.isEmpty()) { + return null; + } // LastAnalyzed is stored per column, but thrift object has it per multiple columns. // Luckily, nobody actually uses it, so we will set to lowest value of all columns for now. ColumnStatisticsDesc desc = StatObjectConverter.getTableColumnStatisticsDesc(mStats.get(0));
