t3rmin4t0r commented on a change in pull request #803: HIVE-14302 URL: https://github.com/apache/hive/pull/803#discussion_r331704100
########## File path: ql/src/test/results/clientpositive/llap/mapjoin_decimal_vectorized.q.out ########## @@ -0,0 +1,686 @@ +PREHOOK: query: CREATE TABLE over1k_n5(t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_n5 +POSTHOOK: query: CREATE TABLE over1k_n5(t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_n5 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/over1k' OVERWRITE INTO TABLE over1k_n5 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over1k_n5 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/over1k' OVERWRITE INTO TABLE over1k_n5 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over1k_n5 +PREHOOK: query: CREATE TABLE t1_n95(`dec` decimal(4,2)) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1_n95 +POSTHOOK: query: CREATE TABLE t1_n95(`dec` decimal(4,2)) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1_n95 +PREHOOK: query: INSERT INTO TABLE t1_n95 select `dec` from over1k_n5 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n5 +PREHOOK: Output: default@t1_n95 +POSTHOOK: query: INSERT INTO TABLE t1_n95 select `dec` from over1k_n5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n5 +POSTHOOK: Output: default@t1_n95 +POSTHOOK: Lineage: t1_n95.dec SIMPLE [(over1k_n5)over1k_n5.FieldSchema(name:dec, type:decimal(4,2), comment:null), ] +PREHOOK: query: CREATE TABLE t2_n59(`dec` decimal(4,0)) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t2_n59 +POSTHOOK: query: CREATE TABLE t2_n59(`dec` decimal(4,0)) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2_n59 +PREHOOK: query: INSERT INTO TABLE t2_n59 select `dec` from over1k_n5 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n5 +PREHOOK: Output: default@t2_n59 +POSTHOOK: query: INSERT INTO TABLE t2_n59 select `dec` from over1k_n5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n5 +POSTHOOK: Output: default@t2_n59 +POSTHOOK: Lineage: t2_n59.dec EXPRESSION [(over1k_n5)over1k_n5.FieldSchema(name:dec, type:decimal(4,2), comment:null), ] +PREHOOK: query: explain vectorization detail select t1_n95.`dec`, t2_n59.`dec` from t1_n95 join t2_n59 on (t1_n95.`dec`=t2_n59.`dec`) order by t1_n95.`dec` +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n95 +PREHOOK: Input: default@t2_n59 +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select t1_n95.`dec`, t2_n59.`dec` from t1_n95 join t2_n59 on (t1_n95.`dec`=t2_n59.`dec`) order by t1_n95.`dec` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n95 +POSTHOOK: Input: default@t2_n59 +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1_n95 + filterExpr: dec is not null (type: boolean) + Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:dec:decimal(4,2)/DECIMAL_64, 1:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:decimal(4,2)/DECIMAL_64) + predicate: dec is not null (type: boolean) + Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: dec (type: decimal(4,2)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: decimal(6,2)) + 1 _col0 (type: decimal(6,2)) + Map Join Vectorization: + bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(6,2)/DECIMAL_64) -> 2:decimal(6,2) + bigTableValueExpressions: col 0:decimal(6,2)/DECIMAL_64 + className: VectorMapJoinOperator + native: false + nativeConditionsMet: hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true + nativeConditionsNotMet: hive.mapjoin.optimized.hashtable IS false + outputColumnNames: _col0, _col1 + input vertices: + 1 Map 3 + Statistics: Num rows: 1112 Data size: 249088 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: decimal(4,2)) + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: 0:decimal(4,2) + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 1:decimal(4,0) + Statistics: Num rows: 1112 Data size: 249088 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(4,0)) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [0] + dataColumns: dec:decimal(4,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(6,2)] + Map 3 + Map Operator Tree: + TableScan + alias: t2_n59 + filterExpr: dec is not null (type: boolean) + Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:dec:decimal(4,0)/DECIMAL_64, 1:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:decimal(4,0)/DECIMAL_64) + predicate: dec is not null (type: boolean) + Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: dec (type: decimal(4,0)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: decimal(6,2)) + sort order: + + Map-reduce partition columns: _col0 (type: decimal(6,2)) + Reduce Sink Vectorization: + className: VectorReduceSinkMultiKeyOperator + keyColumns: 0:decimal(6,2) + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [0] + dataColumns: dec:decimal(4,0)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: z + reduceColumnSortOrder: + + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + dataColumns: KEY.reducesinkkey0:decimal(4,2), VALUE._col0:decimal(4,0) + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: decimal(4,2)), VALUE._col0 (type: decimal(4,0)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 1112 Data size: 249088 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1112 Data size: 249088 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select t1_n95.`dec`, t2_n59.`dec` from t1_n95 join t2_n59 on (t1_n95.`dec`=t2_n59.`dec`) order by t1_n95.`dec` +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n95 +PREHOOK: Input: default@t2_n59 +#### A masked pattern was here #### +POSTHOOK: query: select t1_n95.`dec`, t2_n59.`dec` from t1_n95 join t2_n59 on (t1_n95.`dec`=t2_n59.`dec`) order by t1_n95.`dec` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n95 +POSTHOOK: Input: default@t2_n59 +#### A masked pattern was here #### +14.00 14 +14.00 14 +14.00 14 +14.00 14 +14.00 14 +14.00 14 +14.00 14 +14.00 14 +14.00 14 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +45.00 45 +45.00 45 +45.00 45 +45.00 45 +45.00 45 +6.00 6 +6.00 6 +6.00 6 +6.00 6 +6.00 6 +6.00 6 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +70.00 70 +70.00 70 +70.00 70 +70.00 70 +70.00 70 +70.00 70 +70.00 70 +79.00 79 +79.00 79 +79.00 79 +79.00 79 +79.00 79 +79.00 79 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +PREHOOK: query: select t1_n95.`dec`, t2_n59.`dec` from t1_n95 join t2_n59 on (t1_n95.`dec`=t2_n59.`dec`) order by t1_n95.`dec` +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n95 +PREHOOK: Input: default@t2_n59 +#### A masked pattern was here #### +POSTHOOK: query: select t1_n95.`dec`, t2_n59.`dec` from t1_n95 join t2_n59 on (t1_n95.`dec`=t2_n59.`dec`) order by t1_n95.`dec` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n95 +POSTHOOK: Input: default@t2_n59 +#### A masked pattern was here #### +14.00 14 +14.00 14 +14.00 14 +14.00 14 +14.00 14 +14.00 14 +14.00 14 +14.00 14 +14.00 14 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +17.00 17 +45.00 45 +45.00 45 +45.00 45 +45.00 45 +45.00 45 +6.00 6 +6.00 6 +6.00 6 +6.00 6 +6.00 6 +6.00 6 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +62.00 62 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +64.00 64 +70.00 70 +70.00 70 +70.00 70 +70.00 70 +70.00 70 +70.00 70 +70.00 70 +79.00 79 +79.00 79 +79.00 79 +79.00 79 +79.00 79 +79.00 79 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +89.00 89 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +9.00 9 +PREHOOK: query: explain vectorization detail select t1_n95.`dec`, t2_n59.`dec` from t1_n95 join t2_n59 on (t1_n95.`dec`=t2_n59.`dec`) order by t1_n95.`dec` +PREHOOK: type: QUERY +PREHOOK: Input: default@t1_n95 +PREHOOK: Input: default@t2_n59 +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select t1_n95.`dec`, t2_n59.`dec` from t1_n95 join t2_n59 on (t1_n95.`dec`=t2_n59.`dec`) order by t1_n95.`dec` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1_n95 +POSTHOOK: Input: default@t2_n59 +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1_n95 + filterExpr: dec is not null (type: boolean) + Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:dec:decimal(4,2)/DECIMAL_64, 1:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 0:decimal(4,2)/DECIMAL_64) + predicate: dec is not null (type: boolean) + Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: dec (type: decimal(4,2)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1049 Data size: 117488 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: decimal(6,2)) + 1 _col0 (type: decimal(6,2)) + Map Join Vectorization: + bigTableKeyColumns: 2:decimal(6,2) + bigTableKeyExpressions: ConvertDecimal64ToDecimal(col 0:decimal(6,2)/DECIMAL_64) -> 2:decimal(6,2) Review comment: This is a different perf issue - decimal(6,2) gets converted from decimal64 -> decimal128 type (despite being less than 32 bits) ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
