Repository: hive Updated Branches: refs/heads/master ab33a7b7d -> 18779ea07
HIVE-18678 : fix exim for MM tables and reinstante the test (Sergey Shelukhin, reviewed by Eugene Koifman) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cb225803 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cb225803 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cb225803 Branch: refs/heads/master Commit: cb22580357aeaf8bb1beb4618669d8c03e9f9cc1 Parents: ab33a7b Author: sergey <ser...@apache.org> Authored: Mon Feb 12 11:48:22 2018 -0800 Committer: sergey <ser...@apache.org> Committed: Mon Feb 12 11:48:22 2018 -0800 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 2 +- .../apache/hadoop/hive/ql/exec/CopyTask.java | 4 +- .../apache/hadoop/hive/ql/exec/MoveTask.java | 36 +- .../hive/ql/parse/ImportSemanticAnalyzer.java | 25 +- ql/src/test/queries/clientpositive/mm_exim.q | 8 +- .../results/clientpositive/llap/mm_exim.q.out | 556 +++++++++++++++++++ 6 files changed, 602 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/cb225803/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 974bfac..391170f 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -35,7 +35,6 @@ disabled.query.files=ql_rewrite_gbtoidx.q,\ ql_rewrite_gbtoidx_cbo_2.q,\ rcfile_merge1.q,\ stats_filemetadata.q,\ - mm_exim.q,\ cbo_rp_insert.q,\ cbo_rp_lineage2.q @@ -585,6 +584,7 @@ minillaplocal.query.files=\ mapjoin_hint.q,\ mapjoin_emit_interval.q,\ mergejoin_3way.q,\ + mm_exim.q,\ mrr.q,\ multiMapJoin1.q,\ multiMapJoin2.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/cb225803/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java index 1f5e25f..eee5e66 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.ql.DriverContext; +import org.apache.hadoop.hive.ql.parse.repl.dump.io.FileOperations; import org.apache.hadoop.hive.ql.plan.CopyWork; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.util.StringUtils; @@ -61,6 +62,7 @@ public class CopyTask extends Task<CopyWork> implements Serializable { protected int copyOnePath(Path fromPath, Path toPath) { FileSystem dstFs = null; try { + Utilities.FILE_OP_LOGGER./**/debug("Copying data from {} to {} " + fromPath); console.printInfo("Copying data from " + fromPath.toString(), " to " + toPath.toString()); @@ -85,7 +87,7 @@ public class CopyTask extends Task<CopyWork> implements Serializable { for (FileStatus oneSrc : srcs) { String oneSrcPathStr = oneSrc.getPath().toString(); console.printInfo("Copying file: " + oneSrcPathStr); - LOG.debug("Copying file: {}", oneSrcPathStr); + Utilities.FILE_OP_LOGGER.debug("Copying file {} to {}", oneSrcPathStr, toPath); if (!FileUtils.copy(srcFs, oneSrc.getPath(), dstFs, toPath, false, // delete source true, // overwrite destination http://git-wip-us.apache.org/repos/asf/hive/blob/cb225803/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index 4e804ba..40eb659 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -351,22 +351,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable { // Next we do this for tables and partitions LoadTableDesc tbd = work.getLoadTableWork(); if (tbd != null) { - StringBuilder mesg = new StringBuilder("Loading data to table ") - .append( tbd.getTable().getTableName()); - if (tbd.getPartitionSpec().size() > 0) { - mesg.append(" partition ("); - Map<String, String> partSpec = tbd.getPartitionSpec(); - for (String key: partSpec.keySet()) { - mesg.append(key).append('=').append(partSpec.get(key)).append(", "); - } - mesg.setLength(mesg.length()-2); - mesg.append(')'); - } - String mesg_detail = " from " + tbd.getSourcePath(); - if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { - Utilities.FILE_OP_LOGGER.trace(mesg.toString() + " " + mesg_detail); - } - console.printInfo(mesg.toString(), mesg_detail); + logMessage(tbd); Table table = db.getTable(tbd.getTable().getTableName()); checkFileFormats(db, tbd, table); @@ -452,6 +437,25 @@ public class MoveTask extends Task<MoveWork> implements Serializable { } } + public void logMessage(LoadTableDesc tbd) { + StringBuilder mesg = new StringBuilder("Loading data to table ") + .append( tbd.getTable().getTableName()); + if (tbd.getPartitionSpec().size() > 0) { + mesg.append(" partition ("); + Map<String, String> partSpec = tbd.getPartitionSpec(); + for (String key: partSpec.keySet()) { + mesg.append(key).append('=').append(partSpec.get(key)).append(", "); + } + mesg.setLength(mesg.length()-2); + mesg.append(')'); + } + String mesg_detail = " from " + tbd.getSourcePath(); + if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { + Utilities.FILE_OP_LOGGER.trace(mesg.toString() + " " + mesg_detail); + } + console.printInfo(mesg.toString(), mesg_detail); + } + private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, TaskInformation ti) throws HiveException, IOException, InvalidOperationException { List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec()); http://git-wip-us.apache.org/repos/asf/hive/blob/cb225803/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 70f5e21..5520bc2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -379,8 +379,20 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer { ReplicationSpec replicationSpec, EximUtil.SemanticAnalyzerWrapperContext x, Long txnId, int stmtId, boolean isSourceMm) { Path dataPath = new Path(fromURI.toString(), EximUtil.DATA_PATH_NAME); - Path destPath = !AcidUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtPath) - : new Path(tgtPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId)); + Path destPath = null, loadPath = null; + LoadFileType lft; + if (AcidUtils.isInsertOnlyTable(table)) { + String mmSubdir = replace ? AcidUtils.baseDir(txnId) + : AcidUtils.deltaSubdir(txnId, txnId, stmtId); + destPath = new Path(tgtPath, mmSubdir); + loadPath = tgtPath; + lft = LoadFileType.KEEP_EXISTING; + } else { + destPath = loadPath = x.getCtx().getExternalTmpPath(tgtPath); + lft = replace ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING; + } + + if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("adding import work for table with source location: " + dataPath + "; table: " + tgtPath + "; copy destination " + destPath + "; mm " + txnId + @@ -400,12 +412,10 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer { copyTask = TaskFactory.get(cw, x.getConf()); } - LoadTableDesc loadTableWork = new LoadTableDesc(destPath, - Utilities.getTableDesc(table), new TreeMap<>(), - replace ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING, txnId); + LoadTableDesc loadTableWork = new LoadTableDesc( + loadPath, Utilities.getTableDesc(table), new TreeMap<>(), lft, txnId); loadTableWork.setStmtId(stmtId); - MoveWork mv = new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork, - null, false); + MoveWork mv = new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork, null, false); Task<?> loadTableTask = TaskFactory.get(mv, x.getConf()); copyTask.addDependentTask(loadTableTask); x.getTasks().add(copyTask); @@ -884,6 +894,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer { private static Task<?> createImportCommitTask( String dbName, String tblName, Long txnId, int stmtId, HiveConf conf, boolean isMmTable) { + // TODO: noop, remove? @SuppressWarnings("unchecked") Task<ImportCommitWork> ict = (!isMmTable) ? null : TaskFactory.get( new ImportCommitWork(dbName, tblName, txnId, stmtId), conf); http://git-wip-us.apache.org/repos/asf/hive/blob/cb225803/ql/src/test/queries/clientpositive/mm_exim.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/mm_exim.q b/ql/src/test/queries/clientpositive/mm_exim.q index 8a03f4d..d084125 100644 --- a/ql/src/test/queries/clientpositive/mm_exim.q +++ b/ql/src/test/queries/clientpositive/mm_exim.q @@ -9,7 +9,7 @@ set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; drop table intermediate; -create table intermediate(key int) partitioned by (p int) stored as orc; +create table intermediate(key int) partitioned by (p int) stored as orc tblproperties("transactional"="false"); insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2; insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2; insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2; @@ -17,7 +17,7 @@ insert into table intermediate partition(p='457') select distinct key from src w drop table intermediate_nonpart; drop table intermmediate_part; drop table intermmediate_nonpart; -create table intermediate_nonpart(key int, p int); +create table intermediate_nonpart(key int, p int) tblproperties("transactional"="false"); insert into intermediate_nonpart select * from intermediate; create table intermmediate_nonpart(key int, p int) tblproperties("transactional"="true", "transactional_properties"="insert_only"); insert into intermmediate_nonpart select * from intermediate; @@ -84,13 +84,13 @@ drop table import5_mm; -- MM export into existing non-MM table, non-part and part drop table import6_mm; -create table import6_mm(key int, p int); +create table import6_mm(key int, p int) tblproperties("transactional"="false"); import table import6_mm from 'ql/test/data/exports/intermmediate_nonpart'; select * from import6_mm order by key, p; drop table import6_mm; drop table import7_mm; -create table import7_mm(key int) partitioned by (p int); +create table import7_mm(key int) partitioned by (p int) tblproperties("transactional"="false"); import table import7_mm from 'ql/test/data/exports/intermmediate_part'; select * from import7_mm order by key, p; drop table import7_mm; http://git-wip-us.apache.org/repos/asf/hive/blob/cb225803/ql/src/test/results/clientpositive/llap/mm_exim.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/mm_exim.q.out b/ql/src/test/results/clientpositive/llap/mm_exim.q.out new file mode 100644 index 0000000..1f40754 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/mm_exim.q.out @@ -0,0 +1,556 @@ +PREHOOK: query: drop table intermediate +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table intermediate +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc tblproperties("transactional"="false") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@intermediate +POSTHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc tblproperties("transactional"="false") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@intermediate +PREHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@intermediate@p=455 +POSTHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@intermediate@p=455 +POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@intermediate@p=456 +POSTHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@intermediate@p=456 +POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@intermediate@p=457 +POSTHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@intermediate@p=457 +POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: drop table intermediate_nonpart +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table intermediate_nonpart +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table intermmediate_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table intermmediate_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table intermmediate_nonpart +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table intermmediate_nonpart +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table intermediate_nonpart(key int, p int) tblproperties("transactional"="false") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@intermediate_nonpart +POSTHOOK: query: create table intermediate_nonpart(key int, p int) tblproperties("transactional"="false") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@intermediate_nonpart +PREHOOK: query: insert into intermediate_nonpart select * from intermediate +PREHOOK: type: QUERY +PREHOOK: Input: default@intermediate +PREHOOK: Input: default@intermediate@p=455 +PREHOOK: Input: default@intermediate@p=456 +PREHOOK: Input: default@intermediate@p=457 +PREHOOK: Output: default@intermediate_nonpart +POSTHOOK: query: insert into intermediate_nonpart select * from intermediate +POSTHOOK: type: QUERY +POSTHOOK: Input: default@intermediate +POSTHOOK: Input: default@intermediate@p=455 +POSTHOOK: Input: default@intermediate@p=456 +POSTHOOK: Input: default@intermediate@p=457 +POSTHOOK: Output: default@intermediate_nonpart +POSTHOOK: Lineage: intermediate_nonpart.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: intermediate_nonpart.p SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ] +PREHOOK: query: create table intermmediate_nonpart(key int, p int) tblproperties("transactional"="true", "transactional_properties"="insert_only") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@intermmediate_nonpart +POSTHOOK: query: create table intermmediate_nonpart(key int, p int) tblproperties("transactional"="true", "transactional_properties"="insert_only") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@intermmediate_nonpart +PREHOOK: query: insert into intermmediate_nonpart select * from intermediate +PREHOOK: type: QUERY +PREHOOK: Input: default@intermediate +PREHOOK: Input: default@intermediate@p=455 +PREHOOK: Input: default@intermediate@p=456 +PREHOOK: Input: default@intermediate@p=457 +PREHOOK: Output: default@intermmediate_nonpart +POSTHOOK: query: insert into intermmediate_nonpart select * from intermediate +POSTHOOK: type: QUERY +POSTHOOK: Input: default@intermediate +POSTHOOK: Input: default@intermediate@p=455 +POSTHOOK: Input: default@intermediate@p=456 +POSTHOOK: Input: default@intermediate@p=457 +POSTHOOK: Output: default@intermmediate_nonpart +POSTHOOK: Lineage: intermmediate_nonpart.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: intermmediate_nonpart.p SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ] +PREHOOK: query: create table intermmediate(key int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@intermmediate +POSTHOOK: query: create table intermmediate(key int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@intermmediate +PREHOOK: query: insert into table intermmediate partition(p) select key, p from intermediate +PREHOOK: type: QUERY +PREHOOK: Input: default@intermediate +PREHOOK: Input: default@intermediate@p=455 +PREHOOK: Input: default@intermediate@p=456 +PREHOOK: Input: default@intermediate@p=457 +PREHOOK: Output: default@intermmediate +POSTHOOK: query: insert into table intermmediate partition(p) select key, p from intermediate +POSTHOOK: type: QUERY +POSTHOOK: Input: default@intermediate +POSTHOOK: Input: default@intermediate@p=455 +POSTHOOK: Input: default@intermediate@p=456 +POSTHOOK: Input: default@intermediate@p=457 +POSTHOOK: Output: default@intermmediate@p=455 +POSTHOOK: Output: default@intermmediate@p=456 +POSTHOOK: Output: default@intermmediate@p=457 +POSTHOOK: Lineage: intermmediate PARTITION(p=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: intermmediate PARTITION(p=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: intermmediate PARTITION(p=457).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ] +PREHOOK: query: export table intermediate_nonpart to 'ql/test/data/exports/intermediate_nonpart' +PREHOOK: type: EXPORT +PREHOOK: Input: default@intermediate_nonpart +#### A masked pattern was here #### +POSTHOOK: query: export table intermediate_nonpart to 'ql/test/data/exports/intermediate_nonpart' +POSTHOOK: type: EXPORT +POSTHOOK: Input: default@intermediate_nonpart +#### A masked pattern was here #### +PREHOOK: query: export table intermmediate_nonpart to 'ql/test/data/exports/intermmediate_nonpart' +PREHOOK: type: EXPORT +PREHOOK: Input: default@intermmediate_nonpart +#### A masked pattern was here #### +POSTHOOK: query: export table intermmediate_nonpart to 'ql/test/data/exports/intermmediate_nonpart' +POSTHOOK: type: EXPORT +POSTHOOK: Input: default@intermmediate_nonpart +#### A masked pattern was here #### +PREHOOK: query: export table intermediate to 'ql/test/data/exports/intermediate_part' +PREHOOK: type: EXPORT +PREHOOK: Input: default@intermediate@p=455 +PREHOOK: Input: default@intermediate@p=456 +PREHOOK: Input: default@intermediate@p=457 +#### A masked pattern was here #### +POSTHOOK: query: export table intermediate to 'ql/test/data/exports/intermediate_part' +POSTHOOK: type: EXPORT +POSTHOOK: Input: default@intermediate@p=455 +POSTHOOK: Input: default@intermediate@p=456 +POSTHOOK: Input: default@intermediate@p=457 +#### A masked pattern was here #### +PREHOOK: query: export table intermmediate to 'ql/test/data/exports/intermmediate_part' +PREHOOK: type: EXPORT +PREHOOK: Input: default@intermmediate@p=455 +PREHOOK: Input: default@intermmediate@p=456 +PREHOOK: Input: default@intermmediate@p=457 +#### A masked pattern was here #### +POSTHOOK: query: export table intermmediate to 'ql/test/data/exports/intermmediate_part' +POSTHOOK: type: EXPORT +POSTHOOK: Input: default@intermmediate@p=455 +POSTHOOK: Input: default@intermmediate@p=456 +POSTHOOK: Input: default@intermmediate@p=457 +#### A masked pattern was here #### +PREHOOK: query: drop table intermediate_nonpart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@intermediate_nonpart +PREHOOK: Output: default@intermediate_nonpart +POSTHOOK: query: drop table intermediate_nonpart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@intermediate_nonpart +POSTHOOK: Output: default@intermediate_nonpart +PREHOOK: query: drop table intermmediate_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table intermmediate_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table intermmediate_nonpart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@intermmediate_nonpart +PREHOOK: Output: default@intermmediate_nonpart +POSTHOOK: query: drop table intermmediate_nonpart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@intermmediate_nonpart +POSTHOOK: Output: default@intermmediate_nonpart +PREHOOK: query: drop table import0_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table import0_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table import0_mm(key int, p int) tblproperties("transactional"="true", "transactional_properties"="insert_only") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@import0_mm +POSTHOOK: query: create table import0_mm(key int, p int) tblproperties("transactional"="true", "transactional_properties"="insert_only") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@import0_mm +PREHOOK: query: import table import0_mm from 'ql/test/data/exports/intermediate_nonpart' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: default@import0_mm +POSTHOOK: query: import table import0_mm from 'ql/test/data/exports/intermediate_nonpart' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: default@import0_mm +PREHOOK: query: select * from import0_mm order by key, p +PREHOOK: type: QUERY +PREHOOK: Input: default@import0_mm +#### A masked pattern was here #### +POSTHOOK: query: select * from import0_mm order by key, p +POSTHOOK: type: QUERY +POSTHOOK: Input: default@import0_mm +#### A masked pattern was here #### +0 456 +10 456 +97 455 +98 455 +100 457 +103 457 +PREHOOK: query: drop table import0_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import0_mm +PREHOOK: Output: default@import0_mm +POSTHOOK: query: drop table import0_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import0_mm +POSTHOOK: Output: default@import0_mm +PREHOOK: query: drop table import1_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table import1_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table import1_mm(key int) partitioned by (p int) + stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@import1_mm +POSTHOOK: query: create table import1_mm(key int) partitioned by (p int) + stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@import1_mm +PREHOOK: query: import table import1_mm from 'ql/test/data/exports/intermediate_part' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: default@import1_mm +POSTHOOK: query: import table import1_mm from 'ql/test/data/exports/intermediate_part' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: default@import1_mm +POSTHOOK: Output: default@import1_mm@p=455 +POSTHOOK: Output: default@import1_mm@p=456 +POSTHOOK: Output: default@import1_mm@p=457 +PREHOOK: query: select * from import1_mm order by key, p +PREHOOK: type: QUERY +PREHOOK: Input: default@import1_mm +PREHOOK: Input: default@import1_mm@p=455 +PREHOOK: Input: default@import1_mm@p=456 +PREHOOK: Input: default@import1_mm@p=457 +#### A masked pattern was here #### +POSTHOOK: query: select * from import1_mm order by key, p +POSTHOOK: type: QUERY +POSTHOOK: Input: default@import1_mm +POSTHOOK: Input: default@import1_mm@p=455 +POSTHOOK: Input: default@import1_mm@p=456 +POSTHOOK: Input: default@import1_mm@p=457 +#### A masked pattern was here #### +0 456 +10 456 +97 455 +98 455 +100 457 +103 457 +PREHOOK: query: drop table import1_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import1_mm +PREHOOK: Output: default@import1_mm +POSTHOOK: query: drop table import1_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import1_mm +POSTHOOK: Output: default@import1_mm +PREHOOK: query: drop table import2_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table import2_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: import table import2_mm from 'ql/test/data/exports/intermmediate_nonpart' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: database:default +POSTHOOK: query: import table import2_mm from 'ql/test/data/exports/intermmediate_nonpart' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@import2_mm +PREHOOK: query: desc import2_mm +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@import2_mm +POSTHOOK: query: desc import2_mm +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@import2_mm +key int +p int +PREHOOK: query: select * from import2_mm order by key, p +PREHOOK: type: QUERY +PREHOOK: Input: default@import2_mm +#### A masked pattern was here #### +POSTHOOK: query: select * from import2_mm order by key, p +POSTHOOK: type: QUERY +POSTHOOK: Input: default@import2_mm +#### A masked pattern was here #### +0 456 +10 456 +97 455 +98 455 +100 457 +103 457 +PREHOOK: query: drop table import2_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import2_mm +PREHOOK: Output: default@import2_mm +POSTHOOK: query: drop table import2_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import2_mm +POSTHOOK: Output: default@import2_mm +PREHOOK: query: drop table import3_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table import3_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: import table import3_mm from 'ql/test/data/exports/intermmediate_part' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: database:default +POSTHOOK: query: import table import3_mm from 'ql/test/data/exports/intermmediate_part' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@import3_mm +POSTHOOK: Output: default@import3_mm@p=455 +POSTHOOK: Output: default@import3_mm@p=456 +POSTHOOK: Output: default@import3_mm@p=457 +PREHOOK: query: desc import3_mm +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@import3_mm +POSTHOOK: query: desc import3_mm +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@import3_mm +key int +p int + +# Partition Information +# col_name data_type comment +p int +PREHOOK: query: select * from import3_mm order by key, p +PREHOOK: type: QUERY +PREHOOK: Input: default@import3_mm +PREHOOK: Input: default@import3_mm@p=455 +PREHOOK: Input: default@import3_mm@p=456 +PREHOOK: Input: default@import3_mm@p=457 +#### A masked pattern was here #### +POSTHOOK: query: select * from import3_mm order by key, p +POSTHOOK: type: QUERY +POSTHOOK: Input: default@import3_mm +POSTHOOK: Input: default@import3_mm@p=455 +POSTHOOK: Input: default@import3_mm@p=456 +POSTHOOK: Input: default@import3_mm@p=457 +#### A masked pattern was here #### +0 456 +10 456 +97 455 +98 455 +100 457 +103 457 +PREHOOK: query: drop table import3_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import3_mm +PREHOOK: Output: default@import3_mm +POSTHOOK: query: drop table import3_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import3_mm +POSTHOOK: Output: default@import3_mm +PREHOOK: query: drop table import4_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table import4_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table import4_mm(key int, p int) tblproperties("transactional"="true", "transactional_properties"="insert_only") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@import4_mm +POSTHOOK: query: create table import4_mm(key int, p int) tblproperties("transactional"="true", "transactional_properties"="insert_only") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@import4_mm +PREHOOK: query: import table import4_mm from 'ql/test/data/exports/intermmediate_nonpart' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: default@import4_mm +POSTHOOK: query: import table import4_mm from 'ql/test/data/exports/intermmediate_nonpart' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: default@import4_mm +PREHOOK: query: select * from import4_mm order by key, p +PREHOOK: type: QUERY +PREHOOK: Input: default@import4_mm +#### A masked pattern was here #### +POSTHOOK: query: select * from import4_mm order by key, p +POSTHOOK: type: QUERY +POSTHOOK: Input: default@import4_mm +#### A masked pattern was here #### +0 456 +10 456 +97 455 +98 455 +100 457 +103 457 +PREHOOK: query: drop table import4_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import4_mm +PREHOOK: Output: default@import4_mm +POSTHOOK: query: drop table import4_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import4_mm +POSTHOOK: Output: default@import4_mm +PREHOOK: query: drop table import5_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table import5_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table import5_mm(key int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@import5_mm +POSTHOOK: query: create table import5_mm(key int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@import5_mm +PREHOOK: query: import table import5_mm partition(p=455) from 'ql/test/data/exports/intermmediate_part' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: default@import5_mm +POSTHOOK: query: import table import5_mm partition(p=455) from 'ql/test/data/exports/intermmediate_part' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: default@import5_mm +POSTHOOK: Output: default@import5_mm@p=455 +PREHOOK: query: select * from import5_mm order by key, p +PREHOOK: type: QUERY +PREHOOK: Input: default@import5_mm +PREHOOK: Input: default@import5_mm@p=455 +#### A masked pattern was here #### +POSTHOOK: query: select * from import5_mm order by key, p +POSTHOOK: type: QUERY +POSTHOOK: Input: default@import5_mm +POSTHOOK: Input: default@import5_mm@p=455 +#### A masked pattern was here #### +97 455 +98 455 +PREHOOK: query: drop table import5_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import5_mm +PREHOOK: Output: default@import5_mm +POSTHOOK: query: drop table import5_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import5_mm +POSTHOOK: Output: default@import5_mm +PREHOOK: query: drop table import6_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table import6_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table import6_mm(key int, p int) tblproperties("transactional"="false") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@import6_mm +POSTHOOK: query: create table import6_mm(key int, p int) tblproperties("transactional"="false") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@import6_mm +PREHOOK: query: import table import6_mm from 'ql/test/data/exports/intermmediate_nonpart' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: default@import6_mm +POSTHOOK: query: import table import6_mm from 'ql/test/data/exports/intermmediate_nonpart' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: default@import6_mm +PREHOOK: query: select * from import6_mm order by key, p +PREHOOK: type: QUERY +PREHOOK: Input: default@import6_mm +#### A masked pattern was here #### +POSTHOOK: query: select * from import6_mm order by key, p +POSTHOOK: type: QUERY +POSTHOOK: Input: default@import6_mm +#### A masked pattern was here #### +0 456 +10 456 +97 455 +98 455 +100 457 +103 457 +PREHOOK: query: drop table import6_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import6_mm +PREHOOK: Output: default@import6_mm +POSTHOOK: query: drop table import6_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import6_mm +POSTHOOK: Output: default@import6_mm +PREHOOK: query: drop table import7_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table import7_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table import7_mm(key int) partitioned by (p int) tblproperties("transactional"="false") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@import7_mm +POSTHOOK: query: create table import7_mm(key int) partitioned by (p int) tblproperties("transactional"="false") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@import7_mm +PREHOOK: query: import table import7_mm from 'ql/test/data/exports/intermmediate_part' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: default@import7_mm +POSTHOOK: query: import table import7_mm from 'ql/test/data/exports/intermmediate_part' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: default@import7_mm +POSTHOOK: Output: default@import7_mm@p=455 +POSTHOOK: Output: default@import7_mm@p=456 +POSTHOOK: Output: default@import7_mm@p=457 +PREHOOK: query: select * from import7_mm order by key, p +PREHOOK: type: QUERY +PREHOOK: Input: default@import7_mm +PREHOOK: Input: default@import7_mm@p=455 +PREHOOK: Input: default@import7_mm@p=456 +PREHOOK: Input: default@import7_mm@p=457 +#### A masked pattern was here #### +POSTHOOK: query: select * from import7_mm order by key, p +POSTHOOK: type: QUERY +POSTHOOK: Input: default@import7_mm +POSTHOOK: Input: default@import7_mm@p=455 +POSTHOOK: Input: default@import7_mm@p=456 +POSTHOOK: Input: default@import7_mm@p=457 +#### A masked pattern was here #### +0 456 +10 456 +97 455 +98 455 +100 457 +103 457 +PREHOOK: query: drop table import7_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import7_mm +PREHOOK: Output: default@import7_mm +POSTHOOK: query: drop table import7_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import7_mm +POSTHOOK: Output: default@import7_mm
