Repository: hive Updated Branches: refs/heads/master 93779cb10 -> 4ba12f2f4
HIVE-20343 : Hive 3: CTAS does not respect transactional_properties (Sergey Shelukhin, reviewed by Eugene Koifman) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4ba12f2f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4ba12f2f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4ba12f2f Branch: refs/heads/master Commit: 4ba12f2f47fb3318a139dc64efd7c0eabfca7a4c Parents: 93779cb Author: sergey <[email protected]> Authored: Tue Aug 14 13:15:14 2018 -0700 Committer: sergey <[email protected]> Committed: Tue Aug 14 13:15:14 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 74 ++++++++++++-------- ql/src/test/queries/clientnegative/txnprops.q | 7 ++ .../test/results/clientnegative/txnprops.q.out | 5 ++ 3 files changed, 56 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4ba12f2f/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 2ee562a..6312e74 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -12980,10 +12980,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * property map * @return Modified table property map */ - private Map<String, String> addDefaultProperties( + private Map<String, String> validateAndAddDefaultProperties( Map<String, String> tblProp, boolean isExt, StorageFormat storageFormat, String qualifiedTableName, List<Order> sortCols, boolean isMaterialization, - boolean isTemporaryTable) { + boolean isTemporaryTable) throws SemanticException { Map<String, String> retValue; if (tblProp == null) { retValue = new HashMap<String, String>(); @@ -13002,6 +13002,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } } } + if (!retValue.containsKey(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL) + && retValue.containsKey(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES)) { + throw new SemanticException("Cannot specify " + + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + + " without " + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL); + } if (isExt && HiveConf.getBoolVar(conf, ConfVars.HIVE_EXTERNALTABLE_PURGE_DEFAULT)) { if (retValue.get(MetaStoreUtils.EXTERNAL_TABLE_PURGE) == null) { @@ -13009,7 +13015,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } } - boolean makeInsertOnly = !isTemporaryTable && HiveConf.getBoolVar(conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY); + boolean makeInsertOnly = !isTemporaryTable && HiveConf.getBoolVar( + conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY); boolean makeAcid = !isTemporaryTable && MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID) && HiveConf.getBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY) && @@ -13024,35 +13031,42 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY); } if (makeAcid) { - /*for CTAS, TransactionalValidationListener.makeAcid() runs to late to make table Acid - so the initial write ends up running as non-acid...*/ - try { - Class inputFormatClass = storageFormat.getInputFormat() == null ? null : - Class.forName(storageFormat.getInputFormat()); - Class outputFormatClass = storageFormat.getOutputFormat() == null ? null : - Class.forName(storageFormat.getOutputFormat()); - if (inputFormatClass == null || outputFormatClass == null || - !AcidInputFormat.class.isAssignableFrom(inputFormatClass) || - !AcidOutputFormat.class.isAssignableFrom(outputFormatClass)) { - return retValue; - } - } catch (ClassNotFoundException e) { - LOG.warn("Could not verify InputFormat=" + storageFormat.getInputFormat() + " or OutputFormat=" + - storageFormat.getOutputFormat() + " for " + qualifiedTableName); - return retValue; - } - if(sortCols != null && !sortCols.isEmpty()) { - return retValue; - } - retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); - retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES, - TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY); - LOG.info("Automatically chose to make " + qualifiedTableName + " acid."); + retValue = convertToAcidByDefault(storageFormat, qualifiedTableName, sortCols, retValue); } } return retValue; } + private Map<String, String> convertToAcidByDefault( + StorageFormat storageFormat, String qualifiedTableName, List<Order> sortCols, + Map<String, String> retValue) { + /*for CTAS, TransactionalValidationListener.makeAcid() runs to late to make table Acid + so the initial write ends up running as non-acid...*/ + try { + Class inputFormatClass = storageFormat.getInputFormat() == null ? null : + Class.forName(storageFormat.getInputFormat()); + Class outputFormatClass = storageFormat.getOutputFormat() == null ? null : + Class.forName(storageFormat.getOutputFormat()); + if (inputFormatClass == null || outputFormatClass == null || + !AcidInputFormat.class.isAssignableFrom(inputFormatClass) || + !AcidOutputFormat.class.isAssignableFrom(outputFormatClass)) { + return retValue; + } + } catch (ClassNotFoundException e) { + LOG.warn("Could not verify InputFormat=" + storageFormat.getInputFormat() + " or OutputFormat=" + + storageFormat.getOutputFormat() + " for " + qualifiedTableName); + return retValue; + } + if (sortCols != null && !sortCols.isEmpty()) { + return retValue; + } + retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true"); + retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES, + TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY); + LOG.info("Automatically chose to make " + qualifiedTableName + " acid."); + return retValue; + } + /** * Checks to see if given partition columns has DEFAULT or CHECK constraints (whether ENABLED or DISABLED) * Or has NOT NULL constraints (only ENABLED) @@ -13328,7 +13342,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { throw new SemanticException( "Partition columns can only declared using their name and types in regular CREATE TABLE statements"); } - tblProps = addDefaultProperties( + tblProps = validateAndAddDefaultProperties( tblProps, isExt, storageFormat, dbDotTab, sortCols, isMaterialization, isTemporary); addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, isTemporary, tblProps); @@ -13352,7 +13366,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { break; case CTLT: // create table like <tbl_name> - tblProps = addDefaultProperties( + tblProps = validateAndAddDefaultProperties( tblProps, isExt, storageFormat, dbDotTab, sortCols, isMaterialization, isTemporary); addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, isTemporary, tblProps); @@ -13436,7 +13450,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { "Partition columns can only declared using their names in CTAS statements"); } - tblProps = addDefaultProperties( + tblProps = validateAndAddDefaultProperties( tblProps, isExt, storageFormat, dbDotTab, sortCols, isMaterialization, isTemporary); addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, isTemporary, tblProps); tableDesc = new CreateTableDesc(qualifiedTabName[0], dbDotTab, isExt, isTemporary, cols, http://git-wip-us.apache.org/repos/asf/hive/blob/4ba12f2f/ql/src/test/queries/clientnegative/txnprops.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/txnprops.q b/ql/src/test/queries/clientnegative/txnprops.q new file mode 100644 index 0000000..3c2dcfb --- /dev/null +++ b/ql/src/test/queries/clientnegative/txnprops.q @@ -0,0 +1,7 @@ +--! qt:dataset:src + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +drop table test_props; +create table test_props TBLPROPERTIES ("transactional_properties"="insert_only") as select * from src limit 1; http://git-wip-us.apache.org/repos/asf/hive/blob/4ba12f2f/ql/src/test/results/clientnegative/txnprops.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/txnprops.q.out b/ql/src/test/results/clientnegative/txnprops.q.out new file mode 100644 index 0000000..24a59f3 --- /dev/null +++ b/ql/src/test/results/clientnegative/txnprops.q.out @@ -0,0 +1,5 @@ +PREHOOK: query: drop table test_props +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table test_props +POSTHOOK: type: DROPTABLE +FAILED: SemanticException Cannot specify transactional_properties without transactional
