Repository: hive Updated Branches: refs/heads/master 61372dfaf -> a72693cc3
HIVE-20378 : don't update stats during alter for txn table conversion (Sergey Shelukhin, reviewed by Jason Dere) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a72693cc Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a72693cc Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a72693cc Branch: refs/heads/master Commit: a72693cc3038e7e5e0f33295645c196ab397dd5c Parents: 61372df Author: sergey <[email protected]> Authored: Thu Aug 16 11:49:37 2018 -0700 Committer: sergey <[email protected]> Committed: Thu Aug 16 11:49:37 2018 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/metastore/HiveAlterHandler.java | 4 ++++ .../org/apache/hadoop/hive/metastore/ObjectStore.java | 11 +++++------ .../org/apache/hadoop/hive/metastore/txn/TxnUtils.java | 1 + .../hadoop/hive/metastore/TestHiveAlterHandler.java | 10 ++++++++-- 4 files changed, 18 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a72693cc/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 0441a33..c551b80 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; @@ -812,6 +813,9 @@ public class HiveAlterHandler implements AlterHandler { ColumnStatistics colStats = null; boolean updateColumnStats = !newDbName.equals(dbName) || !newTableName.equals(tableName) || !MetaStoreServerUtils.columnsIncludedByNameType(oldCols, newCols); + // Don't bother in the case of ACID conversion. + updateColumnStats = updateColumnStats + && (TxnUtils.isAcidTable(oldTable) == TxnUtils.isAcidTable(newTable)); if (updateColumnStats) { List<String> oldColNames = new ArrayList<>(oldCols.size()); for (FieldSchema oldCol : oldCols) { http://git-wip-us.apache.org/repos/asf/hive/blob/a72693cc/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 8e2f94e..d27224b 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -4112,7 +4112,8 @@ public class ObjectStore implements RawStore, Configurable { oldt.setDatabase(newt.getDatabase()); oldt.setTableName(normalizeIdentifier(newt.getTableName())); boolean isTxn = TxnUtils.isTransactionalTable(newTable); - if (isTxn && areTxnStatsSupported) { + boolean isToTxn = isTxn && !TxnUtils.isTransactionalTable(oldt.getParameters()); + if (!isToTxn && isTxn && areTxnStatsSupported) { // Transactional table is altered without a txn. Make sure there are no changes to the flag. String errorMsg = verifyStatsChangeCtx(oldt.getParameters(), newTable.getParameters(), newTable.getWriteId(), queryValidWriteIds, false); @@ -4120,7 +4121,6 @@ public class ObjectStore implements RawStore, Configurable { throw new MetaException(errorMsg); } } - boolean isToTxn = isTxn && !TxnUtils.isTransactionalTable(oldt.getParameters()); oldt.setParameters(newt.getParameters()); oldt.setOwner(newt.getOwner()); oldt.setOwnerType(newt.getOwnerType()); @@ -4142,12 +4142,11 @@ public class ObjectStore implements RawStore, Configurable { oldt.setRewriteEnabled(newt.isRewriteEnabled()); // If transactional, update the stats state for the current Stats updater query. - // Don't update for conversion to acid - it doesn't modify stats but passes in qVWIds. - // The fact that it doesn't update stats is verified above. + // Set stats invalid for ACID conversion; it doesn't pass in the write ID. if (isTxn) { - if (!areTxnStatsSupported) { + if (!areTxnStatsSupported || isToTxn) { StatsSetupConst.setBasicStatsState(oldt.getParameters(), StatsSetupConst.FALSE); - } else if (queryValidWriteIds != null && (!isToTxn || newTable.getWriteId() > 0)) { + } else if (queryValidWriteIds != null && newTable.getWriteId() > 0) { // Check concurrent INSERT case and set false to the flag. if (!isCurrentStatsValidForTheQuery(oldt, queryValidWriteIds, true)) { StatsSetupConst.setBasicStatsState(oldt.getParameters(), StatsSetupConst.FALSE); http://git-wip-us.apache.org/repos/asf/hive/blob/a72693cc/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java index aac5811..bd202ed 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java @@ -221,6 +221,7 @@ public class TxnUtils { return false; } Map<String, String> parameters = table.getParameters(); + if (parameters == null) return false; String tableIsTransactional = parameters.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL); return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true"); } http://git-wip-us.apache.org/repos/asf/hive/blob/a72693cc/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java index 8816480..93b2f23 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java @@ -64,7 +64,7 @@ public class TestHiveAlterHandler { } @Test - public void testAlterTableDelColUpdateStats() throws MetaException, InvalidObjectException, NoSuchObjectException { + public void testAlterTableDelColUpdateStats() throws Exception { FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment"); FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment"); FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment"); @@ -85,7 +85,13 @@ public class TestHiveAlterHandler { RawStore msdb = Mockito.mock(RawStore.class); HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); - handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null); + try { + handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null); + } catch (Throwable t) { + System.err.println(t); + t.printStackTrace(System.err); + throw t; + } Mockito.verify(msdb, Mockito.times(1)).getTableColumnStatistics( getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3", "col4") );
