Repository: hive Updated Branches: refs/heads/hive-14535 9ecffcb1b -> c7fb2dbaa
HIVE-14878 : integrate MM tables into ACID: add separate ACID type Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c7fb2dba Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c7fb2dba Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c7fb2dba Branch: refs/heads/hive-14535 Commit: c7fb2dbaa35de53f352a0cc32cfe8312a40fa637 Parents: 9ecffcb Author: Wei Zheng <w...@apache.org> Authored: Tue Oct 18 15:13:20 2016 -0700 Committer: Wei Zheng <w...@apache.org> Committed: Tue Oct 18 15:13:20 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 10 +- .../TransactionalValidationListener.java | 39 ++++--- .../hadoop/hive/ql/exec/FileSinkOperator.java | 15 ++- .../apache/hadoop/hive/ql/exec/MoveTask.java | 3 +- .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 44 ++++++- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 34 +++--- .../queries/clientpositive/mm_insertonly_acid.q | 16 +++ .../clientpositive/mm_insertonly_acid.q.out | 115 +++++++++++++++++++ 8 files changed, 236 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 23314ed..ccc29f8 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1787,10 +1787,12 @@ public class HiveConf extends Configuration { HIVE_TXN_OPERATIONAL_PROPERTIES("hive.txn.operational.properties", 0, "Sets the operational properties that control the appropriate behavior for various\n" - + "versions of the Hive ACID subsystem. Setting it to zero will turn on the legacy mode\n" - + "for ACID, while setting it to one will enable a split-update feature found in the newer\n" - + "version of Hive ACID subsystem. Mostly it is intended to be used as an internal property\n" - + "for future versions of ACID. (See HIVE-14035 for details.)"), + + "versions of the Hive ACID subsystem. Mostly it is intended to be used as an internal property\n" + + "for future versions of ACID. (See HIVE-14035 for details.)\n" + + "0: Turn on the legacy mode for ACID\n" + + "1: Enable split-update feature found in the newer version of Hive ACID subsystem\n" + + "2: Hash-based merge, which combines delta files using GRACE hash join based approach (not implemented)\n" + + "3: Make the table 'quarter-acid' as it only supports insert. But it doesn't require ORC or bucketing."), HIVE_MAX_OPEN_TXNS("hive.max.open.txns", 100000, "Maximum number of open transactions. If \n" + "current open transactions reach this limit, future open transaction requests will be \n" + http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java index 0f08f43..f942479 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java @@ -40,6 +40,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList // These constants are also imported by org.apache.hadoop.hive.ql.io.AcidUtils. public static final String DEFAULT_TRANSACTIONAL_PROPERTY = "default"; public static final String LEGACY_TRANSACTIONAL_PROPERTY = "legacy"; + public static final String INSERTONLY_TRANSACTIONAL_PROPERTY = "insert_only"; TransactionalValidationListener(Configuration conf) { super(conf); @@ -105,8 +106,11 @@ public final class TransactionalValidationListener extends MetaStorePreEventList } if ("true".equalsIgnoreCase(transactionalValue)) { if (!conformToAcid(newTable)) { - throw new MetaException("The table must be bucketed and stored using an ACID compliant" + - " format (such as ORC)"); + // INSERT_ONLY tables don't have to conform to ACID requirement like ORC or bucketing + if (transactionalPropertiesValue == null || !"insert_only".equalsIgnoreCase(transactionalPropertiesValue)) { + throw new MetaException("The table must be bucketed and stored using an ACID compliant" + + " format (such as ORC)"); + } } if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) { @@ -172,32 +176,40 @@ public final class TransactionalValidationListener extends MetaStorePreEventList if (parameters == null || parameters.isEmpty()) { return; } - String transactionalValue = null; - boolean transactionalPropFound = false; + String transactional = null; + String transactionalProperties = null; Set<String> keys = new HashSet<>(parameters.keySet()); for(String key : keys) { - if(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.equalsIgnoreCase(key)) { - transactionalPropFound = true; - transactionalValue = parameters.get(key); + // Get the "transactional" tblproperties value + if (hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.equalsIgnoreCase(key)) { + transactional = parameters.get(key); parameters.remove(key); } + + // Get the "transactional_properties" tblproperties value + if (hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES.equalsIgnoreCase(key)) { + transactionalProperties = parameters.get(key); + } } - if (!transactionalPropFound) { + if (transactional == null) { return; } - if ("false".equalsIgnoreCase(transactionalValue)) { + if ("false".equalsIgnoreCase(transactional)) { // just drop transactional=false. For backward compatibility in case someone has scripts // with transactional=false LOG.info("'transactional'='false' is no longer a valid property and will be ignored"); return; } - if ("true".equalsIgnoreCase(transactionalValue)) { + if ("true".equalsIgnoreCase(transactional)) { if (!conformToAcid(newTable)) { - throw new MetaException("The table must be bucketed and stored using an ACID compliant" + - " format (such as ORC)"); + // INSERT_ONLY tables don't have to conform to ACID requirement like ORC or bucketing + if (transactionalProperties == null || !"insert_only".equalsIgnoreCase(transactionalProperties)) { + throw new MetaException("The table must be bucketed and stored using an ACID compliant" + + " format (such as ORC)"); + } } if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) { @@ -211,7 +223,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList return; } - // transactional prop is found, but the value is not in expected range + // transactional is found, but the value is not in expected range throw new MetaException("'transactional' property of TBLPROPERTIES may only have value 'true'"); } @@ -277,6 +289,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList switch (transactionalProperties) { case DEFAULT_TRANSACTIONAL_PROPERTY: case LEGACY_TRANSACTIONAL_PROPERTY: + case INSERTONLY_TRANSACTIONAL_PROPERTY: isValid = true; break; default: http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index ef6473a..c54187f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -678,7 +678,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), jc); // only create bucket files only if no dynamic partitions, // buckets of dynamic partitions will be created for each newly created partition - if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) { + if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID || + conf.getWriteType() == AcidUtils.Operation.INSERT_ONLY) { fsp.outWriters[filesIdx] = HiveFileFormatUtils.getHiveRecordWriter(jc, conf.getTableInfo(), outputClass, conf, fsp.outPaths[filesIdx], reporter); // If the record writer provides stats, get it from there instead of the serde @@ -821,7 +822,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements // for a given operator branch prediction should work quite nicely on it. // RecordUpdateer expects to get the actual row, not a serialized version of it. Thus we // pass the row rather than recordValue. - if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) { + if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID || + conf.getWriteType() == AcidUtils.Operation.INSERT_ONLY) { rowOutWriters[writerOffset].write(recordValue); } else if (conf.getWriteType() == AcidUtils.Operation.INSERT) { fpaths.updaters[writerOffset].insert(conf.getTransactionId(), row); @@ -865,7 +867,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements protected boolean areAllTrue(boolean[] statsFromRW) { // If we are doing an acid operation they will always all be true as RecordUpdaters always // collect stats - if (conf.getWriteType() != AcidUtils.Operation.NOT_ACID) { + if (conf.getWriteType() != AcidUtils.Operation.NOT_ACID && + conf.getWriteType() != AcidUtils.Operation.INSERT_ONLY) { return true; } for(boolean b : statsFromRW) { @@ -1008,7 +1011,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements // stats from the record writer and store in the previous fsp that is cached if (conf.isGatherStats() && isCollectRWStats) { SerDeStats stats = null; - if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) { + if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID || + conf.getWriteType() == AcidUtils.Operation.INSERT_ONLY) { RecordWriter outWriter = prevFsp.outWriters[0]; if (outWriter != null) { stats = ((StatsProvidingRecordWriter) outWriter).getStats(); @@ -1112,7 +1116,8 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements // record writer already gathers the statistics, it can simply return the // accumulated statistics which will be aggregated in case of spray writers if (conf.isGatherStats() && isCollectRWStats) { - if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) { + if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID || + conf.getWriteType() == AcidUtils.Operation.INSERT_ONLY) { for (int idx = 0; idx < fsp.outWriters.length; idx++) { RecordWriter outWriter = fsp.outWriters[idx]; if (outWriter != null) { http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index f2b8ca3..74a650d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -389,7 +389,8 @@ public class MoveTask extends Task<MoveWork> implements Serializable { db.loadSinglePartition(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(), tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(), - work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID, + (work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID && + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.INSERT_ONLY), hasFollowingStatsTask(), tbd.getMmWriteId()); Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index 7351bbe..ecbc216 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -275,8 +275,9 @@ public class AcidUtils { return result; } + // INSERT_ONLY is a special operation which we only support INSERT operations, no UPDATE/DELETE public enum Operation { - NOT_ACID, INSERT, UPDATE, DELETE; + NOT_ACID, INSERT, UPDATE, DELETE, INSERT_ONLY } /** @@ -344,8 +345,11 @@ public class AcidUtils { public static final String SPLIT_UPDATE_STRING = "split_update"; public static final int HASH_BASED_MERGE_BIT = 0x02; public static final String HASH_BASED_MERGE_STRING = "hash_merge"; + public static final int INSERT_ONLY_BIT = 0x03; + public static final String INSERT_ONLY_STRING = "insert_only"; public static final String DEFAULT_VALUE_STRING = TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY; public static final String LEGACY_VALUE_STRING = TransactionalValidationListener.LEGACY_TRANSACTIONAL_PROPERTY; + public static final String INSERTONLY_VALUE_STRING = TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY; private AcidOperationalProperties() { } @@ -374,6 +378,17 @@ public class AcidUtils { } /** + * Returns an acidOperationalProperties object for tables that uses ACID framework but only + * supports INSERT operation and does not require ORC or bucketing + * @return the acidOperationalProperties object + */ + public static AcidOperationalProperties getInsertOnly() { + AcidOperationalProperties obj = new AcidOperationalProperties(); + obj.setInsertOnly(true); + return obj; + } + + /** * Returns an acidOperationalProperties object that is represented by an encoded string. * @param propertiesStr an encoded string representing the acidOperationalProperties. * @return the acidOperationalProperties object. @@ -388,6 +403,9 @@ public class AcidUtils { if (propertiesStr.equalsIgnoreCase(LEGACY_VALUE_STRING)) { return AcidOperationalProperties.getLegacy(); } + if (propertiesStr.equalsIgnoreCase(INSERTONLY_VALUE_STRING)) { + return AcidOperationalProperties.getInsertOnly(); + } AcidOperationalProperties obj = new AcidOperationalProperties(); String[] options = propertiesStr.split("\\|"); for (String option : options) { @@ -448,6 +466,12 @@ public class AcidUtils { return this; } + public AcidOperationalProperties setInsertOnly(boolean isInsertOnly) { + description = (isInsertOnly + ? (description | INSERT_ONLY_BIT) : (description & ~INSERT_ONLY_BIT)); + return this; + } + public boolean isSplitUpdate() { return (description & SPLIT_UPDATE_BIT) > 0; } @@ -456,6 +480,10 @@ public class AcidUtils { return (description & HASH_BASED_MERGE_BIT) > 0; } + public boolean isInsertOnly() { + return (description & INSERT_ONLY_BIT) > 0; + } + public int toInt() { return description; } @@ -469,6 +497,9 @@ public class AcidUtils { if (isHashBasedMerge()) { str.append("|" + HASH_BASED_MERGE_STRING); } + if (isInsertOnly()) { + str.append("|" + INSERT_ONLY_STRING); + } return str.toString(); } } @@ -1078,6 +1109,17 @@ public class AcidUtils { } /** + * Checks if a table is an ACID table that only supports INSERT, but not UPDATE/DELETE + * @param table table + * @return true if table is an INSERT_ONLY table, false otherwise + */ + public static boolean isInsertOnlyTable(Table table) { + String transactionalProp = table.getProperty(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES); + return transactionalProp != null && + AcidUtils.AcidOperationalProperties.INSERT_ONLY_STRING.equals(transactionalProp); + } + + /** * Sets the acidOperationalProperties in the configuration object argument. * @param conf Mutable configuration object * @param properties An acidOperationalProperties object to initialize from. http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 62faf89..f74c0a9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -6455,7 +6455,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { nullOrder.append(sortOrder == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_ASC ? 'a' : 'z'); } input = genReduceSinkPlan(input, partnCols, sortCols, order.toString(), nullOrder.toString(), - maxReducers, (AcidUtils.isAcidTable(dest_tab) ? getAcidType() : AcidUtils.Operation.NOT_ACID)); + maxReducers, (AcidUtils.isAcidTable(dest_tab) ? + getAcidType(dest_tab, table_desc.getOutputFileFormatClass()) : AcidUtils.Operation.NOT_ACID)); reduceSinkOperatorsAddedByEnforceBucketingSorting.add((ReduceSinkOperator)input.getParentOperators().get(0)); ctx.setMultiFileSpray(multiFileSpray); ctx.setNumFiles(numFiles); @@ -6588,8 +6589,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if (!isNonNativeTable) { AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID; if (destTableIsAcid) { - acidOp = getAcidType(table_desc.getOutputFileFormatClass()); - checkAcidConstraints(qb, table_desc, dest_tab); + acidOp = getAcidType(dest_tab, table_desc.getOutputFileFormatClass()); + checkAcidConstraints(qb, table_desc, dest_tab, acidOp); } try { mmWriteId = getMmWriteId(dest_tab, isMmTable); @@ -6648,8 +6649,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { dest_part.isStoredAsSubDirectories(), conf); AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID; if (destTableIsAcid) { - acidOp = getAcidType(table_desc.getOutputFileFormatClass()); - checkAcidConstraints(qb, table_desc, dest_tab); + acidOp = getAcidType(dest_tab, table_desc.getOutputFileFormatClass()); + checkAcidConstraints(qb, table_desc, dest_tab, acidOp); } try { mmWriteId = getMmWriteId(dest_tab, isMmTable); @@ -6945,7 +6946,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { fileSinkDesc.setHiveServerQuery(SessionState.get().isHiveServerQuery()); // If this is an insert, update, or delete on an ACID table then mark that so the // FileSinkOperator knows how to properly write to it. - if (destTableIsAcid) { + if (destTableIsAcid && !AcidUtils.isInsertOnlyTable(dest_part.getTable())) { AcidUtils.Operation wt = updating() ? AcidUtils.Operation.UPDATE : (deleting() ? AcidUtils.Operation.DELETE : AcidUtils.Operation.INSERT); fileSinkDesc.setWriteType(wt); @@ -7141,7 +7142,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // This method assumes you have already decided that this is an Acid write. Don't call it if // that isn't true. private void checkAcidConstraints(QB qb, TableDesc tableDesc, - Table table) throws SemanticException { + Table table, AcidUtils.Operation acidOp) throws SemanticException { String tableName = tableDesc.getTableName(); if (!qb.getParseInfo().isInsertIntoTable(tableName)) { LOG.debug("Couldn't find table " + tableName + " in insertIntoTable"); @@ -7158,15 +7159,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { */ conf.set(AcidUtils.CONF_ACID_KEY, "true"); - if (table.getNumBuckets() < 1) { - throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, table.getTableName()); - } - if (table.getSortCols() != null && table.getSortCols().size() > 0) { - throw new SemanticException(ErrorMsg.ACID_NO_SORTED_BUCKETS, table.getTableName()); + if (!Operation.NOT_ACID.equals(acidOp) && !Operation.INSERT_ONLY.equals(acidOp)) { + if (table.getNumBuckets() < 1) { + throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, table.getTableName()); + } + if (table.getSortCols() != null && table.getSortCols().size() > 0) { + throw new SemanticException(ErrorMsg.ACID_NO_SORTED_BUCKETS, table.getTableName()); + } } - - - } /** @@ -13118,9 +13118,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { AcidUtils.Operation.INSERT); } - private AcidUtils.Operation getAcidType(Class<? extends OutputFormat> of) { + private AcidUtils.Operation getAcidType(Table table, Class<? extends OutputFormat> of) { if (SessionState.get() == null || !SessionState.get().getTxnMgr().supportsAcid()) { return AcidUtils.Operation.NOT_ACID; + } else if (AcidUtils.isInsertOnlyTable(table)) { + return AcidUtils.Operation.INSERT_ONLY; } else if (isAcidOutputFormat(of)) { return getAcidType(); } else { http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/test/queries/clientpositive/mm_insertonly_acid.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/mm_insertonly_acid.q b/ql/src/test/queries/clientpositive/mm_insertonly_acid.q new file mode 100644 index 0000000..7da99c5 --- /dev/null +++ b/ql/src/test/queries/clientpositive/mm_insertonly_acid.q @@ -0,0 +1,16 @@ +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; +set hive.fetch.task.conversion=none; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + + +drop table qtr_acid; +create table qtr_acid (key int) partitioned by (p int) tblproperties ("transactional"="true", "transactional_properties"="insert_only"); +insert into table qtr_acid partition(p='123') select distinct key from src where key > 0 order by key asc limit 10; +insert into table qtr_acid partition(p='456') select distinct key from src where key > 0 order by key desc limit 10; +explain +select * from qtr_acid order by key; +select * from qtr_acid order by key; +drop table qtr_acid; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/c7fb2dba/ql/src/test/results/clientpositive/mm_insertonly_acid.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/mm_insertonly_acid.q.out b/ql/src/test/results/clientpositive/mm_insertonly_acid.q.out new file mode 100644 index 0000000..6f7d198 --- /dev/null +++ b/ql/src/test/results/clientpositive/mm_insertonly_acid.q.out @@ -0,0 +1,115 @@ +PREHOOK: query: drop table qtr_acid +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table qtr_acid +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table qtr_acid (key int) partitioned by (p int) tblproperties ("transactional"="true", "transactional_properties"="insert_only") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@qtr_acid +POSTHOOK: query: create table qtr_acid (key int) partitioned by (p int) tblproperties ("transactional"="true", "transactional_properties"="insert_only") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@qtr_acid +PREHOOK: query: insert into table qtr_acid partition(p='123') select distinct key from src where key > 0 order by key asc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@qtr_acid@p=123 +POSTHOOK: query: insert into table qtr_acid partition(p='123') select distinct key from src where key > 0 order by key asc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@qtr_acid@p=123 +POSTHOOK: Lineage: qtr_acid PARTITION(p=123).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: insert into table qtr_acid partition(p='456') select distinct key from src where key > 0 order by key desc limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@qtr_acid@p=456 +POSTHOOK: query: insert into table qtr_acid partition(p='456') select distinct key from src where key > 0 order by key desc limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@qtr_acid@p=456 +POSTHOOK: Lineage: qtr_acid PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: explain +select * from qtr_acid order by key +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from qtr_acid order by key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: qtr_acid + Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), p (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 16 Data size: 67 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from qtr_acid order by key +PREHOOK: type: QUERY +PREHOOK: Input: default@qtr_acid +PREHOOK: Input: default@qtr_acid@p=123 +PREHOOK: Input: default@qtr_acid@p=456 +#### A masked pattern was here #### +POSTHOOK: query: select * from qtr_acid order by key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@qtr_acid +POSTHOOK: Input: default@qtr_acid@p=123 +POSTHOOK: Input: default@qtr_acid@p=456 +#### A masked pattern was here #### +9 456 +10 123 +11 123 +85 456 +86 456 +87 456 +90 456 +92 456 +95 456 +96 456 +97 456 +98 456 +100 123 +103 123 +104 123 +105 123 +111 123 +113 123 +114 123 +116 123 +PREHOOK: query: drop table qtr_acid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@qtr_acid +PREHOOK: Output: default@qtr_acid +POSTHOOK: query: drop table qtr_acid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@qtr_acid +POSTHOOK: Output: default@qtr_acid