This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6c57f42879 HIVE-26274: No vectorization if query has upper case window
function (Krisztian Kasa, reviewed by Laszlo Bodor)
6c57f42879 is described below
commit 6c57f4287902aac2e6ca475e9bbf5d3aba68f5c4
Author: Krisztian Kasa <[email protected]>
AuthorDate: Thu Jun 2 08:54:39 2022 +0200
HIVE-26274: No vectorization if query has upper case window function
(Krisztian Kasa, reviewed by Laszlo Bodor)
---
.../hive/ql/optimizer/physical/Vectorizer.java | 2 +-
.../apache/hadoop/hive/ql/plan/VectorPTFDesc.java | 15 ++-----
ql/src/test/queries/clientpositive/vector_ptf_1.q | 4 +-
.../results/clientpositive/llap/llap_smb_ptf.q.out | 4 +-
.../results/clientpositive/llap/semijoin2.q.out | 2 +-
.../results/clientpositive/llap/vector_ptf_1.q.out | 47 ++++++++++++++++++++++
6 files changed, 57 insertions(+), 17 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 01120de64d..ac9dddecbc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -2878,7 +2878,7 @@ public class Vectorizer implements PhysicalPlanResolver {
List<ExprNodeDesc>[] evaluatorInputExprNodeDescLists =
vectorPTFDesc.getEvaluatorInputExprNodeDescLists();
for (int i = 0; i < count; i++) {
- String functionName = evaluatorFunctionNames[i];
+ String functionName = evaluatorFunctionNames[i].toLowerCase();
SupportedFunctionType supportedFunctionType =
VectorPTFDesc.supportedFunctionsMap.get(functionName);
if (supportedFunctionType == null) {
setOperatorIssue(functionName + " not in supported functions " +
VectorPTFDesc.supportedFunctionNames);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
index 0b7f5f3ac7..696c520696 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPTFDesc.java
@@ -119,18 +119,9 @@ public class VectorPTFDesc extends AbstractVectorDesc {
public static HashMap<String, SupportedFunctionType> supportedFunctionsMap =
new HashMap<String, SupportedFunctionType>();
static {
- supportedFunctionsMap.put("row_number", SupportedFunctionType.ROW_NUMBER);
- supportedFunctionsMap.put("rank", SupportedFunctionType.RANK);
- supportedFunctionsMap.put("dense_rank", SupportedFunctionType.DENSE_RANK);
- supportedFunctionsMap.put("min", SupportedFunctionType.MIN);
- supportedFunctionsMap.put("max", SupportedFunctionType.MAX);
- supportedFunctionsMap.put("sum", SupportedFunctionType.SUM);
- supportedFunctionsMap.put("avg", SupportedFunctionType.AVG);
- supportedFunctionsMap.put("first_value",
SupportedFunctionType.FIRST_VALUE);
- supportedFunctionsMap.put("last_value", SupportedFunctionType.LAST_VALUE);
- supportedFunctionsMap.put("count", SupportedFunctionType.COUNT);
- supportedFunctionsMap.put("lead", SupportedFunctionType.LEAD);
- supportedFunctionsMap.put("lag", SupportedFunctionType.LAG);
+ for (SupportedFunctionType supportedFunctionType :
SupportedFunctionType.values()) {
+ supportedFunctionsMap.put(supportedFunctionType.name().toLowerCase(),
supportedFunctionType);
+ }
}
public static List<String> supportedFunctionNames = new ArrayList<String>();
static {
diff --git a/ql/src/test/queries/clientpositive/vector_ptf_1.q
b/ql/src/test/queries/clientpositive/vector_ptf_1.q
index 9bb920e1ca..578cfbdf9f 100644
--- a/ql/src/test/queries/clientpositive/vector_ptf_1.q
+++ b/ql/src/test/queries/clientpositive/vector_ptf_1.q
@@ -20,4 +20,6 @@ group by age, name;
select age, name, avg(gpa), sum(age) over (partition by name)
from studentnull100
-group by age, name;
\ No newline at end of file
+group by age, name;
+
+EXPLAIN VECTORIZATION ONLY SELECT ROW_NUMBER() OVER(order by age) AS rn FROM
studentnull100;
diff --git a/ql/src/test/results/clientpositive/llap/llap_smb_ptf.q.out
b/ql/src/test/results/clientpositive/llap/llap_smb_ptf.q.out
index e31d6c7f53..f9f128b394 100644
--- a/ql/src/test/results/clientpositive/llap/llap_smb_ptf.q.out
+++ b/ql/src/test/results/clientpositive/llap/llap_smb_ptf.q.out
@@ -593,7 +593,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL
Column stats: COMPLETE
Reducer 2
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int),
KEY.reducesinkkey1 (type: string), VALUE._col6 (type: smallint), VALUE._col7
(type: smallint), VALUE._col8 (type: smallint), VALUE._col23 (type: string),
VALUE._col25 (type: string), VALUE._col40 (type: string)
@@ -721,7 +721,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL
Column stats: COMPLETE
Reducer 9
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: string),
KEY.reducesinkkey0 (type: int), VALUE._col2 (type: smallint), VALUE._col3
(type: smallint), VALUE._col5 (type: string), VALUE._col6 (type: string),
VALUE._col8 (type: string)
diff --git a/ql/src/test/results/clientpositive/llap/semijoin2.q.out
b/ql/src/test/results/clientpositive/llap/semijoin2.q.out
index b4eb1ea8df..625363f8c9 100644
--- a/ql/src/test/results/clientpositive/llap/semijoin2.q.out
+++ b/ql/src/test/results/clientpositive/llap/semijoin2.q.out
@@ -160,7 +160,7 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 212 Basic stats: COMPLETE
Column stats: NONE
value expressions: _col16 (type: smallint), _col21 (type:
double), _col99 (type: int)
Reducer 4
- Execution mode: llap
+ Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: VALUE._col16 (type: smallint), VALUE._col21
(type: double), VALUE._col99 (type: int)
diff --git a/ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out
b/ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out
index 407b452aec..ed28e50b2c 100644
--- a/ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out
@@ -296,3 +296,50 @@ NULL sarah ovid 1.62 NULL
26 yuri robinson 1.22 26
69 yuri zipper NULL 69
56 zach robinson 0.86 56
+PREHOOK: query: EXPLAIN VECTORIZATION ONLY SELECT ROW_NUMBER() OVER(order by
age) AS rn FROM studentnull100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@studentnull100
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION ONLY SELECT ROW_NUMBER() OVER(order by
age) AS rn FROM studentnull100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@studentnull100
+#### A masked pattern was here ####
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+
+ Stage: Stage-0
+ Fetch Operator
+
