Repository: hive Updated Branches: refs/heads/branch-2 10f5554e9 -> 029e48b77 refs/heads/master 4ccea29bd -> b8aa16ff6
http://git-wip-us.apache.org/repos/asf/hive/blob/029e48b7/ql/src/test/results/clientpositive/spark/parquet_vectorization_part.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/parquet_vectorization_part.q.out b/ql/src/test/results/clientpositive/spark/parquet_vectorization_part.q.out new file mode 100644 index 0000000..4467b5a --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/parquet_vectorization_part.q.out @@ -0,0 +1,72 @@ +PREHOOK: query: CREATE TABLE alltypesparquet_part(ctinyint tinyint, csmallint smallint, cint int, cbigint bigint, cfloat float, cdouble double, cstring1 string, cstring2 string, ctimestamp1 timestamp, ctimestamp2 timestamp, cboolean1 boolean, cboolean2 boolean) partitioned by (ds string) STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alltypesparquet_part +POSTHOOK: query: CREATE TABLE alltypesparquet_part(ctinyint tinyint, csmallint smallint, cint int, cbigint bigint, cfloat float, cdouble double, cstring1 string, cstring2 string, ctimestamp1 timestamp, ctimestamp2 timestamp, cboolean1 boolean, cboolean2 boolean) partitioned by (ds string) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alltypesparquet_part +PREHOOK: query: insert overwrite table alltypesparquet_part partition (ds='2011') select * from alltypesparquet limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet +PREHOOK: Output: default@alltypesparquet_part@ds=2011 +POSTHOOK: query: insert overwrite table alltypesparquet_part partition (ds='2011') select * from alltypesparquet limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet +POSTHOOK: Output: default@alltypesparquet_part@ds=2011 +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cbigint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cboolean1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cboolean2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cdouble SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cfloat SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).csmallint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cstring1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cstring2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).ctimestamp1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).ctimestamp2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).ctinyint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: insert overwrite table alltypesparquet_part partition (ds='2012') select * from alltypesparquet limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet +PREHOOK: Output: default@alltypesparquet_part@ds=2012 +POSTHOOK: query: insert overwrite table alltypesparquet_part partition (ds='2012') select * from alltypesparquet limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet +POSTHOOK: Output: default@alltypesparquet_part@ds=2012 +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cbigint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cboolean1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cboolean2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cdouble SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cfloat SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).csmallint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cstring1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cstring2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).ctimestamp1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).ctimestamp2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).ctinyint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: select count(cdouble), cint from alltypesparquet_part where ds='2011' group by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet_part +PREHOOK: Input: default@alltypesparquet_part@ds=2011 +#### A masked pattern was here #### +POSTHOOK: query: select count(cdouble), cint from alltypesparquet_part where ds='2011' group by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet_part +POSTHOOK: Input: default@alltypesparquet_part@ds=2011 +#### A masked pattern was here #### +100 528534767 +PREHOOK: query: select count(*) from alltypesparquet_part A join alltypesparquet_part B on A.ds=B.ds +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet_part +PREHOOK: Input: default@alltypesparquet_part@ds=2011 +PREHOOK: Input: default@alltypesparquet_part@ds=2012 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from alltypesparquet_part A join alltypesparquet_part B on A.ds=B.ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet_part +POSTHOOK: Input: default@alltypesparquet_part@ds=2011 +POSTHOOK: Input: default@alltypesparquet_part@ds=2012 +#### A masked pattern was here #### +20000 http://git-wip-us.apache.org/repos/asf/hive/blob/029e48b7/ql/src/test/results/clientpositive/spark/parquet_vectorization_part_project.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/parquet_vectorization_part_project.q.out b/ql/src/test/results/clientpositive/spark/parquet_vectorization_part_project.q.out new file mode 100644 index 0000000..944101f --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/parquet_vectorization_part_project.q.out @@ -0,0 +1,143 @@ +PREHOOK: query: CREATE TABLE alltypesparquet_part(ctinyint tinyint, csmallint smallint, cint int, cbigint bigint, cfloat float, cdouble double, cstring1 string, cstring2 string, ctimestamp1 timestamp, ctimestamp2 timestamp, cboolean1 boolean, cboolean2 boolean) partitioned by (ds string) STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alltypesparquet_part +POSTHOOK: query: CREATE TABLE alltypesparquet_part(ctinyint tinyint, csmallint smallint, cint int, cbigint bigint, cfloat float, cdouble double, cstring1 string, cstring2 string, ctimestamp1 timestamp, ctimestamp2 timestamp, cboolean1 boolean, cboolean2 boolean) partitioned by (ds string) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alltypesparquet_part +PREHOOK: query: insert overwrite table alltypesparquet_part partition (ds='2011') select * from alltypesparquet order by ctinyint, cint, cbigint limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet +PREHOOK: Output: default@alltypesparquet_part@ds=2011 +POSTHOOK: query: insert overwrite table alltypesparquet_part partition (ds='2011') select * from alltypesparquet order by ctinyint, cint, cbigint limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet +POSTHOOK: Output: default@alltypesparquet_part@ds=2011 +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cbigint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cboolean1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cboolean2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cdouble SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cfloat SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).csmallint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cstring1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).cstring2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).ctimestamp1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).ctimestamp2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2011).ctinyint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: insert overwrite table alltypesparquet_part partition (ds='2012') select * from alltypesparquet order by ctinyint, cint, cbigint limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet +PREHOOK: Output: default@alltypesparquet_part@ds=2012 +POSTHOOK: query: insert overwrite table alltypesparquet_part partition (ds='2012') select * from alltypesparquet order by ctinyint, cint, cbigint limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet +POSTHOOK: Output: default@alltypesparquet_part@ds=2012 +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cbigint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cboolean1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cboolean2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cdouble SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cfloat SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).csmallint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cstring1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).cstring2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).ctimestamp1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).ctimestamp2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part PARTITION(ds=2012).ctinyint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: explain vectorization select (cdouble+2) c1 from alltypesparquet_part order by c1 limit 10 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization select (cdouble+2) c1 from alltypesparquet_part order by c1 limit 10 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesparquet_part + Statistics: Num rows: 200 Data size: 2400 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: (cdouble + 2.0) (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 200 Data size: 2400 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Statistics: Num rows: 200 Data size: 2400 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: double) + outputColumnNames: _col0 + Statistics: Num rows: 200 Data size: 2400 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + ListSink + +PREHOOK: query: select (cdouble+2) c1 from alltypesparquet_part order by c1 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet_part +PREHOOK: Input: default@alltypesparquet_part@ds=2011 +PREHOOK: Input: default@alltypesparquet_part@ds=2012 +#### A masked pattern was here #### +POSTHOOK: query: select (cdouble+2) c1 from alltypesparquet_part order by c1 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet_part +POSTHOOK: Input: default@alltypesparquet_part@ds=2011 +POSTHOOK: Input: default@alltypesparquet_part@ds=2012 +#### A masked pattern was here #### +NULL +NULL +-15863.0 +-15863.0 +-14988.0 +-14988.0 +-14646.0 +-14646.0 +-14236.0 +-14236.0 http://git-wip-us.apache.org/repos/asf/hive/blob/029e48b7/ql/src/test/results/clientpositive/spark/parquet_vectorization_part_varchar.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/parquet_vectorization_part_varchar.q.out b/ql/src/test/results/clientpositive/spark/parquet_vectorization_part_varchar.q.out new file mode 100644 index 0000000..1208217 --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/parquet_vectorization_part_varchar.q.out @@ -0,0 +1,72 @@ +PREHOOK: query: CREATE TABLE alltypesparquet_part_varchar(ctinyint tinyint, csmallint smallint, cint int, cbigint bigint, cfloat float, cdouble double, cstring1 string, cstring2 string, ctimestamp1 timestamp, ctimestamp2 timestamp, cboolean1 boolean, cboolean2 boolean) partitioned by (ds varchar(4)) STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alltypesparquet_part_varchar +POSTHOOK: query: CREATE TABLE alltypesparquet_part_varchar(ctinyint tinyint, csmallint smallint, cint int, cbigint bigint, cfloat float, cdouble double, cstring1 string, cstring2 string, ctimestamp1 timestamp, ctimestamp2 timestamp, cboolean1 boolean, cboolean2 boolean) partitioned by (ds varchar(4)) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alltypesparquet_part_varchar +PREHOOK: query: insert overwrite table alltypesparquet_part_varchar partition (ds='2011') select * from alltypesparquet limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet +PREHOOK: Output: default@alltypesparquet_part_varchar@ds=2011 +POSTHOOK: query: insert overwrite table alltypesparquet_part_varchar partition (ds='2011') select * from alltypesparquet limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet +POSTHOOK: Output: default@alltypesparquet_part_varchar@ds=2011 +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).cbigint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).cboolean1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).cboolean2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).cdouble SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).cfloat SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).cint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).csmallint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).cstring1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).cstring2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).ctimestamp1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).ctimestamp2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2011).ctinyint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: insert overwrite table alltypesparquet_part_varchar partition (ds='2012') select * from alltypesparquet limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet +PREHOOK: Output: default@alltypesparquet_part_varchar@ds=2012 +POSTHOOK: query: insert overwrite table alltypesparquet_part_varchar partition (ds='2012') select * from alltypesparquet limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet +POSTHOOK: Output: default@alltypesparquet_part_varchar@ds=2012 +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).cbigint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cbigint, type:bigint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).cboolean1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean1, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).cboolean2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cboolean2, type:boolean, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).cdouble SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).cfloat SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).cint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).csmallint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:csmallint, type:smallint, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).cstring1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).cstring2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:cstring2, type:string, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).ctimestamp1 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).ctimestamp2 SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] +POSTHOOK: Lineage: alltypesparquet_part_varchar PARTITION(ds=2012).ctinyint SIMPLE [(alltypesparquet)alltypesparquet.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] +PREHOOK: query: select count(cdouble), cint from alltypesparquet_part_varchar where ds='2011' group by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet_part_varchar +PREHOOK: Input: default@alltypesparquet_part_varchar@ds=2011 +#### A masked pattern was here #### +POSTHOOK: query: select count(cdouble), cint from alltypesparquet_part_varchar where ds='2011' group by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet_part_varchar +POSTHOOK: Input: default@alltypesparquet_part_varchar@ds=2011 +#### A masked pattern was here #### +100 528534767 +PREHOOK: query: select count(*) from alltypesparquet_part_varchar A join alltypesparquet_part_varchar B on A.ds=B.ds +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet_part_varchar +PREHOOK: Input: default@alltypesparquet_part_varchar@ds=2011 +PREHOOK: Input: default@alltypesparquet_part_varchar@ds=2012 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from alltypesparquet_part_varchar A join alltypesparquet_part_varchar B on A.ds=B.ds +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet_part_varchar +POSTHOOK: Input: default@alltypesparquet_part_varchar@ds=2011 +POSTHOOK: Input: default@alltypesparquet_part_varchar@ds=2012 +#### A masked pattern was here #### +20000 http://git-wip-us.apache.org/repos/asf/hive/blob/029e48b7/ql/src/test/results/clientpositive/spark/parquet_vectorization_pushdown.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/parquet_vectorization_pushdown.q.out b/ql/src/test/results/clientpositive/spark/parquet_vectorization_pushdown.q.out new file mode 100644 index 0000000..a95898f --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/parquet_vectorization_pushdown.q.out @@ -0,0 +1,90 @@ +WARNING: Comparing a bigint and a double may result in a loss of precision. +PREHOOK: query: explain vectorization SELECT AVG(cbigint) FROM alltypesparquet WHERE cbigint < cdouble +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization SELECT AVG(cbigint) FROM alltypesparquet WHERE cbigint < cdouble +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesparquet + filterExpr: (UDFToDouble(cbigint) < cdouble) (type: boolean) + Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(cbigint) < cdouble) (type: boolean) + Statistics: Num rows: 4096 Data size: 49152 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cbigint (type: bigint) + outputColumnNames: cbigint + Statistics: Num rows: 4096 Data size: 49152 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: avg(cbigint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Group By Operator + aggregations: avg(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +WARNING: Comparing a bigint and a double may result in a loss of precision. +PREHOOK: query: SELECT AVG(cbigint) FROM alltypesparquet WHERE cbigint < cdouble +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesparquet +#### A masked pattern was here #### +POSTHOOK: query: SELECT AVG(cbigint) FROM alltypesparquet WHERE cbigint < cdouble +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesparquet +#### A masked pattern was here #### +-1.4670720493864927E9 http://git-wip-us.apache.org/repos/asf/hive/blob/029e48b7/ql/src/test/results/clientpositive/spark/temp_table.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/temp_table.q.out b/ql/src/test/results/clientpositive/spark/temp_table.q.out index c2ec3b4..8521002 100644 --- a/ql/src/test/results/clientpositive/spark/temp_table.q.out +++ b/ql/src/test/results/clientpositive/spark/temp_table.q.out @@ -374,6 +374,7 @@ POSTHOOK: query: SHOW TABLES POSTHOOK: type: SHOWTABLES POSTHOOK: Input: database:default alltypesorc +alltypesparquet bar bay baz
