Repository: hive Updated Branches: refs/heads/master 00e8826ba -> c31c9bf1e
HIVE-18422 : Vectorized input format should not be used when input format is excluded and row.serde is enabled (Vihang Karajgaonkar, reviewed by Matt McCline) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c31c9bf1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c31c9bf1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c31c9bf1 Branch: refs/heads/master Commit: c31c9bf1e592b7c085a9cfe2879b92b9a198cbaa Parents: 00e8826 Author: Vihang Karajgaonkar <[email protected]> Authored: Wed Jan 10 14:18:12 2018 -0800 Committer: Vihang Karajgaonkar <[email protected]> Committed: Tue Jan 23 16:50:00 2018 -0800 ---------------------------------------------------------------------- .../hive/ql/optimizer/physical/Vectorizer.java | 21 +- .../vectorization_input_format_excludes.q | 24 +++ .../vectorization_input_format_excludes.q.out | 198 +++++++++++++++++-- .../vectorization_input_format_excludes.q.out | 164 +++++++++++++++ 4 files changed, 389 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c31c9bf1/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 9002a20..215aaad 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -524,7 +524,9 @@ public class Vectorizer implements PhysicalPlanResolver { int partitionColumnCount; List<VirtualColumn> availableVirtualColumnList; List<VirtualColumn> neededVirtualColumnList; - boolean useVectorizedInputFileFormat; + //not to be confused with useVectorizedInputFileFormat at Vectorizer level + //which represents the value of configuration hive.vectorized.use.vectorized.input.format + private boolean useVectorizedInputFileFormat; Set<Support> inputFormatSupportSet; Set<Support> supportSetInUse; @@ -1299,6 +1301,20 @@ public class Vectorizer implements PhysicalPlanResolver { return false; } + private boolean shouldUseVectorizedInputFormat(Set<String> inputFileFormatClassNames) { + if (inputFileFormatClassNames == null || inputFileFormatClassNames.isEmpty() + || !useVectorizedInputFileFormat) { + return useVectorizedInputFileFormat; + } + //Global config of vectorized input format is enabled; check if these inputformats are excluded + for (String inputFormat : inputFileFormatClassNames) { + if(isInputFormatExcluded(inputFormat, vectorizedInputFormatExcludes)) { + return false; + } + } + return true; + } + private boolean isInputFormatExcluded(String inputFileFormatClassName, Collection<Class<?>> excludes) { Class<?> ifClass = null; try { @@ -1494,7 +1510,8 @@ public class Vectorizer implements PhysicalPlanResolver { vectorTaskColumnInfo.setDataColumnNums(dataColumnNums); vectorTaskColumnInfo.setPartitionColumnCount(partitionColumnCount); vectorTaskColumnInfo.setAvailableVirtualColumnList(availableVirtualColumnList); - vectorTaskColumnInfo.setUseVectorizedInputFileFormat(useVectorizedInputFileFormat); + vectorTaskColumnInfo.setUseVectorizedInputFileFormat( + shouldUseVectorizedInputFormat(inputFileFormatClassNameSet)); vectorTaskColumnInfo.setInputFormatSupportSet(inputFormatSupportSet); http://git-wip-us.apache.org/repos/asf/hive/blob/c31c9bf1/ql/src/test/queries/clientpositive/vectorization_input_format_excludes.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/vectorization_input_format_excludes.q b/ql/src/test/queries/clientpositive/vectorization_input_format_excludes.q index f8e8245..8d22efd 100644 --- a/ql/src/test/queries/clientpositive/vectorization_input_format_excludes.q +++ b/ql/src/test/queries/clientpositive/vectorization_input_format_excludes.q @@ -167,3 +167,27 @@ select ctinyint, stddev_pop(cdouble) from alltypes_orc group by ctinyint; + +-- test when input format is excluded row serde is used for vectorization +set hive.vectorized.input.format.excludes=org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat,org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; +set hive.vectorized.use.vectorized.input.format=true; +set hive.vectorized.use.row.serde.deserialize=true; +set hive.vectorized.row.serde.inputformat.excludes=; + +create table orcTbl (t1 tinyint, t2 tinyint) +stored as orc; + +insert into orcTbl values (54, 9), (-104, 25), (-112, 24); + +explain vectorization select t1, t2, (t1+t2) from orcTbl where (t1+t2) > 10; + +select t1, t2, (t1+t2) from orcTbl where (t1+t2) > 10; + +create table parquetTbl (t1 tinyint, t2 tinyint) +stored as parquet; + +insert into parquetTbl values (54, 9), (-104, 25), (-112, 24); + +explain vectorization SELECT t1, t2, (t1 + t2) FROM parquetTbl WHERE (t1 + t2) > 10; + +SELECT t1, t2, (t1 + t2) FROM parquetTbl WHERE (t1 + t2) > 10; http://git-wip-us.apache.org/repos/asf/hive/blob/c31c9bf1/ql/src/test/results/clientpositive/llap/vectorization_input_format_excludes.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vectorization_input_format_excludes.q.out b/ql/src/test/results/clientpositive/llap/vectorization_input_format_excludes.q.out index ab8752a..516b1c4 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_input_format_excludes.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_input_format_excludes.q.out @@ -115,14 +115,14 @@ PREHOOK: query: select * limit 10 PREHOOK: type: QUERY PREHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from alltypes_parquet where cint = 528534767 limit 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 528534767 -11 -15431 -11.0 -15431.0 cvLH6Eat2yFsyy7p 528534767 -28 -15813 -28.0 -15813.0 cvLH6Eat2yFsyy7p 528534767 -34 15007 -34.0 15007.0 cvLH6Eat2yFsyy7p @@ -238,7 +238,7 @@ PREHOOK: query: select ctinyint, group by ctinyint PREHOOK: type: QUERY PREHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select ctinyint, max(cint), min(csmallint), @@ -249,7 +249,7 @@ POSTHOOK: query: select ctinyint, group by ctinyint POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### -1 626923679 -15441 36 -1.0486250072717667 8786.246963933321 -10 626923679 -15384 28 -10.0 8850.451610567823 -11 626923679 -15659 32 -11.0 10453.738567408038 @@ -442,14 +442,14 @@ PREHOOK: query: select * limit 10 PREHOOK: type: QUERY PREHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from alltypes_parquet where cint = 528534767 limit 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 528534767 -11 -15431 -11.0 -15431.0 cvLH6Eat2yFsyy7p 528534767 -28 -15813 -28.0 -15813.0 cvLH6Eat2yFsyy7p 528534767 -34 15007 -34.0 15007.0 cvLH6Eat2yFsyy7p @@ -560,7 +560,7 @@ PREHOOK: query: select ctinyint, group by ctinyint PREHOOK: type: QUERY PREHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select ctinyint, max(cint), min(csmallint), @@ -571,7 +571,7 @@ POSTHOOK: query: select ctinyint, group by ctinyint POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### -1 626923679 -15441 36 -1.0486250072717667 8786.246963933321 -10 626923679 -15384 28 -10.0 8850.451610567823 -11 626923679 -15659 32 -11.0 10453.738567408038 @@ -769,14 +769,14 @@ PREHOOK: query: select * limit 10 PREHOOK: type: QUERY PREHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from alltypes_parquet where cint = 528534767 limit 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 528534767 -11 -15431 -11.0 -15431.0 cvLH6Eat2yFsyy7p 528534767 -28 -15813 -28.0 -15813.0 cvLH6Eat2yFsyy7p 528534767 -34 15007 -34.0 15007.0 cvLH6Eat2yFsyy7p @@ -892,7 +892,7 @@ PREHOOK: query: select ctinyint, group by ctinyint PREHOOK: type: QUERY PREHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select ctinyint, max(cint), min(csmallint), @@ -903,7 +903,7 @@ POSTHOOK: query: select ctinyint, group by ctinyint POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypes_parquet -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### -1 626923679 -15441 36 -1.0486250072717667 8786.246963933321 -10 626923679 -15384 28 -10.0 8850.451610567823 -11 626923679 -15659 32 -11.0 10453.738567408038 @@ -1144,14 +1144,14 @@ PREHOOK: query: select * limit 10 PREHOOK: type: QUERY PREHOOK: Input: default@alltypes_orc -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select * from alltypes_orc where cint = 528534767 limit 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypes_orc -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### 528534767 -11 -15431 -11.0 -15431.0 cvLH6Eat2yFsyy7p 528534767 -28 -15813 -28.0 -15813.0 cvLH6Eat2yFsyy7p 528534767 -34 15007 -34.0 15007.0 cvLH6Eat2yFsyy7p @@ -1262,7 +1262,7 @@ PREHOOK: query: select ctinyint, group by ctinyint PREHOOK: type: QUERY PREHOOK: Input: default@alltypes_orc -#### A masked pattern was here #### +PREHOOK: Output: hdfs://### HDFS PATH ### POSTHOOK: query: select ctinyint, max(cint), min(csmallint), @@ -1273,7 +1273,7 @@ POSTHOOK: query: select ctinyint, group by ctinyint POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypes_orc -#### A masked pattern was here #### +POSTHOOK: Output: hdfs://### HDFS PATH ### -1 626923679 -15441 36 -1.0486250072717667 8786.246963933321 -10 626923679 -15384 28 -10.0 8850.451610567823 -11 626923679 -15659 32 -11.0 10453.738567408038 @@ -1402,3 +1402,169 @@ POSTHOOK: Input: default@alltypes_orc 8 1070764888 -15778 1034 8.0 9562.355155774725 9 626923679 -13629 25 9.0 10157.217948808622 NULL 1073418988 -16379 3115 NULL 305051.4870777435 +PREHOOK: query: create table orcTbl (t1 tinyint, t2 tinyint) +stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orcTbl +POSTHOOK: query: create table orcTbl (t1 tinyint, t2 tinyint) +stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcTbl +PREHOOK: query: insert into orcTbl values (54, 9), (-104, 25), (-112, 24) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@orctbl +POSTHOOK: query: insert into orcTbl values (54, 9), (-104, 25), (-112, 24) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@orctbl +POSTHOOK: Lineage: orctbl.t1 SCRIPT [] +POSTHOOK: Lineage: orctbl.t2 SCRIPT [] +PREHOOK: query: explain vectorization select t1, t2, (t1+t2) from orcTbl where (t1+t2) > 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization select t1, t2, (t1+t2) from orcTbl where (t1+t2) > 10 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orctbl + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToInteger((t1 + t2)) > 10) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: t1 (type: tinyint), t2 (type: tinyint), (t1 + t2) (type: tinyint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select t1, t2, (t1+t2) from orcTbl where (t1+t2) > 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orctbl +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select t1, t2, (t1+t2) from orcTbl where (t1+t2) > 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orctbl +POSTHOOK: Output: hdfs://### HDFS PATH ### +54 9 63 +PREHOOK: query: create table parquetTbl (t1 tinyint, t2 tinyint) +stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@parquetTbl +POSTHOOK: query: create table parquetTbl (t1 tinyint, t2 tinyint) +stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquetTbl +PREHOOK: query: insert into parquetTbl values (54, 9), (-104, 25), (-112, 24) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@parquettbl +POSTHOOK: query: insert into parquetTbl values (54, 9), (-104, 25), (-112, 24) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@parquettbl +POSTHOOK: Lineage: parquettbl.t1 SCRIPT [] +POSTHOOK: Lineage: parquettbl.t2 SCRIPT [] +PREHOOK: query: explain vectorization SELECT t1, t2, (t1 + t2) FROM parquetTbl WHERE (t1 + t2) > 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization SELECT t1, t2, (t1 + t2) FROM parquetTbl WHERE (t1 + t2) > 10 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: parquettbl + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToInteger((t1 + t2)) > 10) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: t1 (type: tinyint), t2 (type: tinyint), (t1 + t2) (type: tinyint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs (cache only) + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT t1, t2, (t1 + t2) FROM parquetTbl WHERE (t1 + t2) > 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@parquettbl +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT t1, t2, (t1 + t2) FROM parquetTbl WHERE (t1 + t2) > 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquettbl +POSTHOOK: Output: hdfs://### HDFS PATH ### +54 9 63 http://git-wip-us.apache.org/repos/asf/hive/blob/c31c9bf1/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out b/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out index 887f44b..80b4f6f 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out @@ -1386,3 +1386,167 @@ POSTHOOK: Input: default@alltypes_orc 8 1070764888 -15778 1034 8.0 9562.355155774725 9 626923679 -13629 25 9.0 10157.217948808622 NULL 1073418988 -16379 3115 NULL 305051.4870777435 +PREHOOK: query: create table orcTbl (t1 tinyint, t2 tinyint) +stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orcTbl +POSTHOOK: query: create table orcTbl (t1 tinyint, t2 tinyint) +stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orcTbl +PREHOOK: query: insert into orcTbl values (54, 9), (-104, 25), (-112, 24) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@orctbl +POSTHOOK: query: insert into orcTbl values (54, 9), (-104, 25), (-112, 24) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@orctbl +POSTHOOK: Lineage: orctbl.t1 SCRIPT [] +POSTHOOK: Lineage: orctbl.t2 SCRIPT [] +PREHOOK: query: explain vectorization select t1, t2, (t1+t2) from orcTbl where (t1+t2) > 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization select t1, t2, (t1+t2) from orcTbl where (t1+t2) > 10 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: orctbl + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToInteger((t1 + t2)) > 10) (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: t1 (type: tinyint), t2 (type: tinyint), (t1 + t2) (type: tinyint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select t1, t2, (t1+t2) from orcTbl where (t1+t2) > 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@orctbl +#### A masked pattern was here #### +POSTHOOK: query: select t1, t2, (t1+t2) from orcTbl where (t1+t2) > 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orctbl +#### A masked pattern was here #### +54 9 63 +PREHOOK: query: create table parquetTbl (t1 tinyint, t2 tinyint) +stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@parquetTbl +POSTHOOK: query: create table parquetTbl (t1 tinyint, t2 tinyint) +stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquetTbl +PREHOOK: query: insert into parquetTbl values (54, 9), (-104, 25), (-112, 24) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@parquettbl +POSTHOOK: query: insert into parquetTbl values (54, 9), (-104, 25), (-112, 24) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@parquettbl +POSTHOOK: Lineage: parquettbl.t1 SCRIPT [] +POSTHOOK: Lineage: parquettbl.t2 SCRIPT [] +PREHOOK: query: explain vectorization SELECT t1, t2, (t1 + t2) FROM parquetTbl WHERE (t1 + t2) > 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization SELECT t1, t2, (t1 + t2) FROM parquetTbl WHERE (t1 + t2) > 10 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: parquettbl + Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToInteger((t1 + t2)) > 10) (type: boolean) + Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: t1 (type: tinyint), t2 (type: tinyint), (t1 + t2) (type: tinyint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT t1, t2, (t1 + t2) FROM parquetTbl WHERE (t1 + t2) > 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@parquettbl +#### A masked pattern was here #### +POSTHOOK: query: SELECT t1, t2, (t1 + t2) FROM parquetTbl WHERE (t1 + t2) > 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquettbl +#### A masked pattern was here #### +54 9 63
