Repository: hive Updated Branches: refs/heads/master f9efd84f8 -> 47f45705f
HIVE-18599: Transactions: Fix CTAS on Micromanaged tables (Steve Yeom, reviewed by Gopal V) Signed-off-by: Gopal V <gop...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/47f45705 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/47f45705 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/47f45705 Branch: refs/heads/master Commit: 47f45705f997658b3ce5df4276915a6c0be56d8a Parents: f9efd84 Author: Steve Yeom <sy...@hortonworks.com> Authored: Fri Feb 2 16:48:02 2018 -0800 Committer: Gopal V <gop...@apache.org> Committed: Fri Feb 2 16:50:23 2018 -0800 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 1 + .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 13 ++- ql/src/test/queries/clientpositive/mm_cttas.q | 20 +++++ .../results/clientpositive/llap/mm_cttas.q.out | 89 ++++++++++++++++++++ 4 files changed, 120 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/47f45705/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 70d0749..fed9394 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -71,6 +71,7 @@ minillap.shared.query.files=insert_into1.q,\ mapreduce1.q,\ mapreduce2.q,\ mm_all.q,\ + mm_cttas.q,\ orc_merge1.q,\ orc_merge10.q,\ orc_merge2.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/47f45705/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index dbf9363..c2e2499 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2053,8 +2053,15 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } } try { - fname = ctx.getExtTmpPathRelTo( - FileUtils.makeQualified(location, conf)).toString(); + CreateTableDesc tblDesc = qb.getTableDesc(); + if (tblDesc != null + && tblDesc.isTemporary() + && AcidUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) { + fname = FileUtils.makeQualified(location, conf).toString(); + } else { + fname = ctx.getExtTmpPathRelTo( + FileUtils.makeQualified(location, conf)).toString(); + } } catch (Exception e) { throw new SemanticException(generateErrorMessage(ast, "Error creating temporary folder on: " + location.toString()), e); @@ -6842,7 +6849,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { field_schemas = new ArrayList<FieldSchema>(); destTableIsTemporary = tblDesc.isTemporary(); destTableIsMaterialization = tblDesc.isMaterialization(); - if (!destTableIsTemporary && AcidUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) { + if (AcidUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) { isMmTable = isMmCtas = true; txnId = SessionState.get().getTxnMgr().getCurrentTxnId(); tblDesc.setInitialMmWriteId(txnId); http://git-wip-us.apache.org/repos/asf/hive/blob/47f45705/ql/src/test/queries/clientpositive/mm_cttas.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/mm_cttas.q b/ql/src/test/queries/clientpositive/mm_cttas.q new file mode 100644 index 0000000..b099d2f --- /dev/null +++ b/ql/src/test/queries/clientpositive/mm_cttas.q @@ -0,0 +1,20 @@ + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +drop table intermediate; +create table intermediate(key int) partitioned by (p int) stored as orc; +insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2; +insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2; +insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2; + +drop table cttas1_mm; + +create temporary table cttas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate; + +select * from cttas1_mm; + +drop table cttas1_mm; + +drop table intermediate; + http://git-wip-us.apache.org/repos/asf/hive/blob/47f45705/ql/src/test/results/clientpositive/llap/mm_cttas.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/mm_cttas.q.out b/ql/src/test/results/clientpositive/llap/mm_cttas.q.out new file mode 100644 index 0000000..01acec5 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/mm_cttas.q.out @@ -0,0 +1,89 @@ +PREHOOK: query: drop table intermediate +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table intermediate +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@intermediate +POSTHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@intermediate +PREHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@intermediate@p=455 +POSTHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@intermediate@p=455 +POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@intermediate@p=456 +POSTHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@intermediate@p=456 +POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@intermediate@p=457 +POSTHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@intermediate@p=457 +POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: drop table cttas1_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table cttas1_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: create temporary table cttas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@intermediate +PREHOOK: Input: default@intermediate@p=455 +PREHOOK: Input: default@intermediate@p=456 +PREHOOK: Input: default@intermediate@p=457 +PREHOOK: Output: database:default +PREHOOK: Output: default@cttas1_mm +POSTHOOK: query: create temporary table cttas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@intermediate +POSTHOOK: Input: default@intermediate@p=455 +POSTHOOK: Input: default@intermediate@p=456 +POSTHOOK: Input: default@intermediate@p=457 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cttas1_mm +PREHOOK: query: select * from cttas1_mm +PREHOOK: type: QUERY +PREHOOK: Input: default@cttas1_mm +#### A masked pattern was here #### +POSTHOOK: query: select * from cttas1_mm +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cttas1_mm +#### A masked pattern was here #### +98 455 +97 455 +100 457 +103 457 +0 456 +10 456 +PREHOOK: query: drop table cttas1_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@cttas1_mm +PREHOOK: Output: default@cttas1_mm +POSTHOOK: query: drop table cttas1_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@cttas1_mm +POSTHOOK: Output: default@cttas1_mm +PREHOOK: query: drop table intermediate +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@intermediate +PREHOOK: Output: default@intermediate +POSTHOOK: query: drop table intermediate +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@intermediate +POSTHOOK: Output: default@intermediate
