HIVE-15020 : handle truncate for MM tables (not atomic yet) (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ff1ea20c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ff1ea20c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ff1ea20c Branch: refs/heads/hive-14535 Commit: ff1ea20cb72e7e754fca1fc00e712adec257a204 Parents: 8004f71 Author: Sergey Shelukhin <[email protected]> Authored: Thu Oct 20 15:07:39 2016 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Thu Oct 20 15:07:39 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/exec/TaskFactory.java | 1 + .../hive/ql/parse/DDLSemanticAnalyzer.java | 9 +- .../queries/clientnegative/mm_truncate_cols.q | 3 + ql/src/test/queries/clientpositive/mm_all.q | 20 ++- .../clientnegative/mm_truncate_cols.q.out | 9 ++ .../results/clientpositive/llap/mm_all.q.out | 152 +++++++++++++------ 6 files changed, 136 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java index 822ff41..1cd20e3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java @@ -147,6 +147,7 @@ public final class TaskFactory { throw new RuntimeException("No task for work class " + workClass.getName()); } + @SafeVarargs public static <T extends Serializable> Task<T> get(T work, HiveConf conf, Task<? extends Serializable>... tasklist) { Task<T> ret = get((Class<T>) work.getClass(), conf); http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index a018b54..038cbbf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -963,6 +963,10 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { if (indexes != null && indexes.size() > 0) { throw new SemanticException(ErrorMsg.TRUNCATE_COLUMN_INDEXED_TABLE.getMsg()); } + // It would be possible to support this, but this is such a pointless command. + if (MetaStoreUtils.isMmTable(table.getParameters())) { + throw new SemanticException("Truncating MM table columns not presently supported"); + } List<String> bucketCols = null; Class<? extends InputFormat> inputFormatClass = null; @@ -1060,12 +1064,11 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { // so the operation is atomic. Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc); truncateTblDesc.setOutputDir(queryTmpdir); - // TODO# movetask is created here; handle MM tables LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc, partSpec == null ? new HashMap<String, String>() : partSpec, null); ltd.setLbCtx(lbCtx); - Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false), - conf); + @SuppressWarnings("unchecked") + Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false), conf); truncateTask.addDependentTask(moveTsk); // Recalculate the HDFS stats if auto gather stats is set http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/test/queries/clientnegative/mm_truncate_cols.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/mm_truncate_cols.q b/ql/src/test/queries/clientnegative/mm_truncate_cols.q new file mode 100644 index 0000000..1780118 --- /dev/null +++ b/ql/src/test/queries/clientnegative/mm_truncate_cols.q @@ -0,0 +1,3 @@ +CREATE TABLE mm_table(key int, value string) tblproperties('hivecommit'='true'); + +TRUNCATE TABLE mm_table COLUMNS (value); http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/test/queries/clientpositive/mm_all.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/mm_all.q b/ql/src/test/queries/clientpositive/mm_all.q index 6639aaa..d440136 100644 --- a/ql/src/test/queries/clientpositive/mm_all.q +++ b/ql/src/test/queries/clientpositive/mm_all.q @@ -21,6 +21,10 @@ insert into table part_mm partition(key_mm='455') select key from intermediate; insert into table part_mm partition(key_mm='456') select key from intermediate; insert into table part_mm partition(key_mm='455') select key from intermediate; select * from part_mm order by key, key_mm; +truncate table part_mm partition(key_mm='455'); +select * from part_mm order by key, key_mm; +truncate table part_mm; +select * from part_mm order by key, key_mm; drop table part_mm; drop table simple_mm; @@ -30,6 +34,8 @@ insert overwrite table simple_mm select key from intermediate; select * from simple_mm order by key; insert into table simple_mm select key from intermediate; select * from simple_mm order by key; +truncate table simple_mm; +select * from simple_mm; drop table simple_mm; @@ -201,9 +207,9 @@ drop table iow0_mm; create table iow0_mm(key int) tblproperties('hivecommit'='true'); insert overwrite table iow0_mm select key from intermediate; insert into table iow0_mm select key + 1 from intermediate; -select * from iow0_mm; +select * from iow0_mm order by key; insert overwrite table iow0_mm select key + 2 from intermediate; -select * from iow0_mm; +select * from iow0_mm order by key; drop table iow0_mm; @@ -213,13 +219,13 @@ insert overwrite table iow1_mm partition (key2) select key as k1, key from intermediate union all select key as k1, key from intermediate; insert into table iow1_mm partition (key2) select key + 1 as k1, key from intermediate union all select key as k1, key from intermediate; -select * from iow1_mm; +select * from iow1_mm order by key, key2; insert overwrite table iow1_mm partition (key2) select key + 3 as k1, key from intermediate union all select key + 4 as k1, key from intermediate; -select * from iow1_mm; +select * from iow1_mm order by key, key2; insert overwrite table iow1_mm partition (key2) select key + 3 as k1, key + 3 from intermediate union all select key + 2 as k1, key + 2 from intermediate; -select * from iow1_mm; +select * from iow1_mm order by key, key2; drop table iow1_mm; @@ -351,12 +357,10 @@ import table import7_mm from 'ql/test/data/exports/intermmediate_part'; select * from import7_mm order by key, p; drop table import7_mm; - - set hive.exim.test.mode=false; --- TODO# multi-insert, truncate +-- TODO# multi-insert http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/test/results/clientnegative/mm_truncate_cols.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/mm_truncate_cols.q.out b/ql/src/test/results/clientnegative/mm_truncate_cols.q.out new file mode 100644 index 0000000..015f251 --- /dev/null +++ b/ql/src/test/results/clientnegative/mm_truncate_cols.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: CREATE TABLE mm_table(key int, value string) tblproperties('hivecommit'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@mm_table +POSTHOOK: query: CREATE TABLE mm_table(key int, value string) tblproperties('hivecommit'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mm_table +FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Truncating MM table columns not presently supported http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/test/results/clientpositive/llap/mm_all.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/mm_all.q.out b/ql/src/test/results/clientpositive/llap/mm_all.q.out index 7d77047..bdd365d 100644 --- a/ql/src/test/results/clientpositive/llap/mm_all.q.out +++ b/ql/src/test/results/clientpositive/llap/mm_all.q.out @@ -180,6 +180,50 @@ POSTHOOK: Input: default@part_mm@key_mm=456 103 455 103 455 103 456 +PREHOOK: query: truncate table part_mm partition(key_mm='455') +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@part_mm@key_mm=455 +POSTHOOK: query: truncate table part_mm partition(key_mm='455') +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@part_mm@key_mm=455 +PREHOOK: query: select * from part_mm order by key, key_mm +PREHOOK: type: QUERY +PREHOOK: Input: default@part_mm +PREHOOK: Input: default@part_mm@key_mm=455 +PREHOOK: Input: default@part_mm@key_mm=456 +#### A masked pattern was here #### +POSTHOOK: query: select * from part_mm order by key, key_mm +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_mm +POSTHOOK: Input: default@part_mm@key_mm=455 +POSTHOOK: Input: default@part_mm@key_mm=456 +#### A masked pattern was here #### +0 456 +10 456 +97 456 +98 456 +100 456 +103 456 +PREHOOK: query: truncate table part_mm +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@part_mm@key_mm=455 +PREHOOK: Output: default@part_mm@key_mm=456 +POSTHOOK: query: truncate table part_mm +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@part_mm@key_mm=455 +POSTHOOK: Output: default@part_mm@key_mm=456 +PREHOOK: query: select * from part_mm order by key, key_mm +PREHOOK: type: QUERY +PREHOOK: Input: default@part_mm +PREHOOK: Input: default@part_mm@key_mm=455 +PREHOOK: Input: default@part_mm@key_mm=456 +#### A masked pattern was here #### +POSTHOOK: query: select * from part_mm order by key, key_mm +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_mm +POSTHOOK: Input: default@part_mm@key_mm=455 +POSTHOOK: Input: default@part_mm@key_mm=456 +#### A masked pattern was here #### PREHOOK: query: drop table part_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@part_mm @@ -279,6 +323,20 @@ POSTHOOK: Input: default@simple_mm 100 103 103 +PREHOOK: query: truncate table simple_mm +PREHOOK: type: TRUNCATETABLE +PREHOOK: Output: default@simple_mm +POSTHOOK: query: truncate table simple_mm +POSTHOOK: type: TRUNCATETABLE +POSTHOOK: Output: default@simple_mm +PREHOOK: query: select * from simple_mm +PREHOOK: type: QUERY +PREHOOK: Input: default@simple_mm +#### A masked pattern was here #### +POSTHOOK: query: select * from simple_mm +POSTHOOK: type: QUERY +POSTHOOK: Input: default@simple_mm +#### A masked pattern was here #### PREHOOK: query: drop table simple_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@simple_mm @@ -1299,26 +1357,26 @@ POSTHOOK: Input: default@intermediate@p=456 POSTHOOK: Input: default@intermediate@p=457 POSTHOOK: Output: default@iow0_mm POSTHOOK: Lineage: iow0_mm.key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] -PREHOOK: query: select * from iow0_mm +PREHOOK: query: select * from iow0_mm order by key PREHOOK: type: QUERY PREHOOK: Input: default@iow0_mm #### A masked pattern was here #### -POSTHOOK: query: select * from iow0_mm +POSTHOOK: query: select * from iow0_mm order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@iow0_mm #### A masked pattern was here #### -98 -97 -100 -103 0 +1 10 -99 +11 +97 98 +98 +99 +100 101 +103 104 -1 -11 PREHOOK: query: insert overwrite table iow0_mm select key + 2 from intermediate PREHOOK: type: QUERY PREHOOK: Input: default@intermediate @@ -1334,20 +1392,20 @@ POSTHOOK: Input: default@intermediate@p=456 POSTHOOK: Input: default@intermediate@p=457 POSTHOOK: Output: default@iow0_mm POSTHOOK: Lineage: iow0_mm.key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] -PREHOOK: query: select * from iow0_mm +PREHOOK: query: select * from iow0_mm order by key PREHOOK: type: QUERY PREHOOK: Input: default@iow0_mm #### A masked pattern was here #### -POSTHOOK: query: select * from iow0_mm +POSTHOOK: query: select * from iow0_mm order by key POSTHOOK: type: QUERY POSTHOOK: Input: default@iow0_mm #### A masked pattern was here #### -100 +2 +12 99 +100 102 105 -2 -12 PREHOOK: query: drop table iow0_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@iow0_mm @@ -1422,7 +1480,7 @@ POSTHOOK: Lineage: iow1_mm PARTITION(key2=103).key EXPRESSION [(intermediate)int POSTHOOK: Lineage: iow1_mm PARTITION(key2=10).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: iow1_mm PARTITION(key2=97).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: iow1_mm PARTITION(key2=98).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] -PREHOOK: query: select * from iow1_mm +PREHOOK: query: select * from iow1_mm order by key, key2 PREHOOK: type: QUERY PREHOOK: Input: default@iow1_mm PREHOOK: Input: default@iow1_mm@key2=0 @@ -1432,7 +1490,7 @@ PREHOOK: Input: default@iow1_mm@key2=103 PREHOOK: Input: default@iow1_mm@key2=97 PREHOOK: Input: default@iow1_mm@key2=98 #### A masked pattern was here #### -POSTHOOK: query: select * from iow1_mm +POSTHOOK: query: select * from iow1_mm order by key, key2 POSTHOOK: type: QUERY POSTHOOK: Input: default@iow1_mm POSTHOOK: Input: default@iow1_mm@key2=0 @@ -1444,28 +1502,28 @@ POSTHOOK: Input: default@iow1_mm@key2=98 #### A masked pattern was here #### 0 0 0 0 -1 0 0 0 +1 0 10 10 10 10 -11 10 10 10 -100 100 -100 100 -101 100 -100 100 -103 103 -103 103 -104 103 -103 103 +11 10 97 97 97 97 -98 97 97 97 +98 97 98 98 98 98 -99 98 98 98 +99 98 +100 100 +100 100 +100 100 +101 100 +103 103 +103 103 +103 103 +104 103 PREHOOK: query: insert overwrite table iow1_mm partition (key2) select key + 3 as k1, key from intermediate union all select key + 4 as k1, key from intermediate PREHOOK: type: QUERY @@ -1493,7 +1551,7 @@ POSTHOOK: Lineage: iow1_mm PARTITION(key2=103).key EXPRESSION [(intermediate)int POSTHOOK: Lineage: iow1_mm PARTITION(key2=10).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: iow1_mm PARTITION(key2=97).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: iow1_mm PARTITION(key2=98).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] -PREHOOK: query: select * from iow1_mm +PREHOOK: query: select * from iow1_mm order by key, key2 PREHOOK: type: QUERY PREHOOK: Input: default@iow1_mm PREHOOK: Input: default@iow1_mm@key2=0 @@ -1503,7 +1561,7 @@ PREHOOK: Input: default@iow1_mm@key2=103 PREHOOK: Input: default@iow1_mm@key2=97 PREHOOK: Input: default@iow1_mm@key2=98 #### A masked pattern was here #### -POSTHOOK: query: select * from iow1_mm +POSTHOOK: query: select * from iow1_mm order by key, key2 POSTHOOK: type: QUERY POSTHOOK: Input: default@iow1_mm POSTHOOK: Input: default@iow1_mm@key2=0 @@ -1517,14 +1575,14 @@ POSTHOOK: Input: default@iow1_mm@key2=98 4 0 13 10 14 10 -103 100 -104 100 -106 103 -107 103 100 97 101 97 101 98 102 98 +103 100 +104 100 +106 103 +107 103 PREHOOK: query: insert overwrite table iow1_mm partition (key2) select key + 3 as k1, key + 3 from intermediate union all select key + 2 as k1, key + 2 from intermediate PREHOOK: type: QUERY @@ -1562,7 +1620,7 @@ POSTHOOK: Lineage: iow1_mm PARTITION(key2=13).key EXPRESSION [(intermediate)inte POSTHOOK: Lineage: iow1_mm PARTITION(key2=2).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: iow1_mm PARTITION(key2=3).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: iow1_mm PARTITION(key2=99).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] -PREHOOK: query: select * from iow1_mm +PREHOOK: query: select * from iow1_mm order by key, key2 PREHOOK: type: QUERY PREHOOK: Input: default@iow1_mm PREHOOK: Input: default@iow1_mm@key2=0 @@ -1581,7 +1639,7 @@ PREHOOK: Input: default@iow1_mm@key2=97 PREHOOK: Input: default@iow1_mm@key2=98 PREHOOK: Input: default@iow1_mm@key2=99 #### A masked pattern was here #### -POSTHOOK: query: select * from iow1_mm +POSTHOOK: query: select * from iow1_mm order by key, key2 POSTHOOK: type: QUERY POSTHOOK: Input: default@iow1_mm POSTHOOK: Input: default@iow1_mm@key2=0 @@ -1600,26 +1658,26 @@ POSTHOOK: Input: default@iow1_mm@key2=97 POSTHOOK: Input: default@iow1_mm@key2=98 POSTHOOK: Input: default@iow1_mm@key2=99 #### A masked pattern was here #### +2 2 3 0 +3 3 4 0 +12 12 13 10 +13 13 14 10 +99 99 +100 97 100 100 100 100 +101 97 +101 98 101 101 +102 98 102 102 103 103 105 105 106 106 -12 12 -13 13 -2 2 -3 3 -100 97 -101 97 -101 98 -102 98 -99 99 PREHOOK: query: drop table iow1_mm PREHOOK: type: DROPTABLE PREHOOK: Input: default@iow1_mm @@ -2458,7 +2516,7 @@ POSTHOOK: query: drop table import7_mm POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@import7_mm POSTHOOK: Output: default@import7_mm -PREHOOK: query: -- TODO# multi-insert, truncate +PREHOOK: query: -- TODO# multi-insert @@ -2466,7 +2524,7 @@ drop table intermediate PREHOOK: type: DROPTABLE PREHOOK: Input: default@intermediate PREHOOK: Output: default@intermediate -POSTHOOK: query: -- TODO# multi-insert, truncate +POSTHOOK: query: -- TODO# multi-insert
