This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d2d7ff14eef HIVE-29696: Vectorization: ClassCastException in
VectorPTFGroupBatches when buffering complex window function inputs (#6579)
d2d7ff14eef is described below
commit d2d7ff14eefebacf296008bb5d03f620fcc3cd72
Author: Raghav Aggarwal <[email protected]>
AuthorDate: Mon Jul 13 18:20:10 2026 +0530
HIVE-29696: Vectorization: ClassCastException in VectorPTFGroupBatches when
buffering complex window function inputs (#6579)
---
.../vector/ptf/BufferedVectorizedRowBatch.java | 1 -
.../ql/exec/vector/ptf/VectorPTFEvaluatorBase.java | 6 -
.../exec/vector/ptf/VectorPTFEvaluatorCount.java | 1 -
.../ptf/VectorPTFEvaluatorCountDistinct.java | 1 -
.../vector/ptf/VectorPTFEvaluatorCountStar.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDecimalAvg.java | 1 -
.../ptf/VectorPTFEvaluatorDecimalFirstValue.java | 1 -
.../ptf/VectorPTFEvaluatorDecimalLastValue.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDecimalMax.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDecimalMin.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDecimalSum.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDoubleAvg.java | 1 -
.../ptf/VectorPTFEvaluatorDoubleFirstValue.java | 1 -
.../ptf/VectorPTFEvaluatorDoubleLastValue.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDoubleMax.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDoubleMin.java | 1 -
.../vector/ptf/VectorPTFEvaluatorDoubleSum.java | 1 -
.../ql/exec/vector/ptf/VectorPTFEvaluatorLag.java | 2 +-
.../ql/exec/vector/ptf/VectorPTFEvaluatorLead.java | 2 +-
.../exec/vector/ptf/VectorPTFEvaluatorLongAvg.java | 1 -
.../ptf/VectorPTFEvaluatorLongFirstValue.java | 1 -
.../ptf/VectorPTFEvaluatorLongLastValue.java | 1 -
.../exec/vector/ptf/VectorPTFEvaluatorLongMax.java | 1 -
.../exec/vector/ptf/VectorPTFEvaluatorLongMin.java | 1 -
.../exec/vector/ptf/VectorPTFEvaluatorLongSum.java | 1 -
.../vector/ptf/VectorPTFEvaluatorRowNumber.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDecimalAvg.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDecimalMax.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDecimalMin.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDecimalSum.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDoubleAvg.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDoubleMax.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDoubleMin.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingDoubleSum.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingLongAvg.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingLongMax.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingLongMin.java | 1 -
.../ptf/VectorPTFEvaluatorStreamingLongSum.java | 1 -
.../ql/exec/vector/ptf/VectorPTFGroupBatches.java | 12 -
.../hive/ql/exec/vector/ptf/VectorPTFOperator.java | 61 ++-
.../exec/vector/ptf/VectorSpillBlockContainer.java | 5 -
.../clientpositive/vector_ptf_nested_input_expr.q | 56 +++
.../llap/vector_ptf_nested_input_expr.q.out | 434 +++++++++++++++++++++
43 files changed, 522 insertions(+), 91 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java
index f70fc2dea47..2e750f6408d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java
@@ -24,7 +24,6 @@
*/
public class BufferedVectorizedRowBatch extends VectorizedRowBatch {
boolean isLastGroupBatch;
- boolean isInputExpressionEvaluated;
public BufferedVectorizedRowBatch(int numCols) {
super(numCols);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java
index 9243c3cb00e..31995c434d8 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java
@@ -86,12 +86,6 @@ public void setNullsLast(boolean nullsLast) {
this.nullsLast = nullsLast;
}
- // Evaluate the aggregation input argument expression.
- public void evaluateInputExpr(VectorizedRowBatch batch) throws HiveException
{
- if (inputVecExpr != null) {
- inputVecExpr.evaluate(batch);
- }
- }
/*
* Evaluate the aggregation over one of the group's batches. In this mode,
evaluator is not
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java
index 94b038e2e8e..eed564b3971 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java
@@ -46,7 +46,6 @@ public VectorPTFEvaluatorCount(WindowFrameDef windowFrameDef,
VectorExpression i
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Count non-null column rows.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java
index 167c07dc209..3f34274cf01 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java
@@ -68,7 +68,6 @@ public VectorPTFEvaluatorCountDistinct(WindowFrameDef
windowFrameDef,
@Override
public void evaluateGroupBatch(VectorizedRowBatch batch) throws
HiveException {
- evaluateInputExpr(batch);
// We do not filter when PTF is in reducer.
Preconditions.checkState(!batch.selectedInUse);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java
index c48141ef8d1..3cff308550c 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java
@@ -42,7 +42,6 @@ public VectorPTFEvaluatorCountStar(WindowFrameDef
windowFrameDef, VectorExpressi
@Override
public void evaluateGroupBatch(VectorizedRowBatch batch) {
// No input expression for COUNT(*).
- // evaluateInputExpr(batch);
// Count all rows.
count += batch.size;
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java
index 5d4d757060d..7f7cfec9d0d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java
@@ -54,7 +54,6 @@ public VectorPTFEvaluatorDecimalAvg(WindowFrameDef
windowFrameDef, VectorExpress
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null decimal column values for avg; maintain
isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java
index d21e882084b..c500cf193ec 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java
@@ -50,7 +50,6 @@ public VectorPTFEvaluatorDecimalFirstValue(WindowFrameDef
windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// First row determines isGroupResultNull and decimal firstValue; stream
fill result as repeated.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java
index 0fb88d69c3d..78ff8b7bd20 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java
@@ -50,7 +50,6 @@ public VectorPTFEvaluatorDecimalLastValue(WindowFrameDef
windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Last row of last batch determines isGroupResultNull and decimal
lastValue.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java
index 11f146db706..33a8a8b4ddb 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorDecimalMax(WindowFrameDef
windowFrameDef, VectorExpress
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null decimal column values; maintain
isGroupResultNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java
index e06d1868856..bb57159338f 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java
@@ -46,7 +46,6 @@ public VectorPTFEvaluatorDecimalMin(WindowFrameDef
windowFrameDef, VectorExpress
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null decimal column values; maintain
isGroupResultNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java
index f9263e17af9..02b388d55dc 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorDecimalSum(WindowFrameDef
windowFrameDef, VectorExpress
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null decimal column values; maintain isGroupResultNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java
index d7123fb2579..7cd42a382c2 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java
@@ -46,7 +46,6 @@ public VectorPTFEvaluatorDoubleAvg(WindowFrameDef
windowFrameDef, VectorExpressi
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null double column values for avg; maintain
isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java
index 40b49d7960c..026baa0aa33 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java
@@ -49,7 +49,6 @@ public VectorPTFEvaluatorDoubleFirstValue(WindowFrameDef
windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// First row determines isGroupResultNull and double firstValue; stream
fill result as repeated.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java
index e7e4a0ce3a7..d8936413678 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java
@@ -47,7 +47,6 @@ public VectorPTFEvaluatorDoubleLastValue(WindowFrameDef
windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Last row of last batch determines isGroupResultNull and double
lastValue.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java
index 5b124d20269..1a8f029639d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorDoubleMax(WindowFrameDef
windowFrameDef, VectorExpressi
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null double column values; maintain
isGroupResultNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java
index b76df013b35..e8f28eb9c31 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorDoubleMin(WindowFrameDef
windowFrameDef, VectorExpressi
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null double column values; maintain
isGroupResultNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java
index e8cda454bcd..4ecefb4f2fe 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java
@@ -42,7 +42,6 @@ public VectorPTFEvaluatorDoubleSum(WindowFrameDef
windowFrameDef, VectorExpressi
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null double column values; maintain isGroupResultNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java
index d35d089c6c0..31922508862 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java
@@ -38,7 +38,7 @@ public Object runOnRange(int rowNum, Range range,
VectorPTFGroupBatches batches)
if (lagRow < 0 || lagRow >= batches.size()) {
return getDefaultValue(rowNum, batches);
} else {
- return batches.getValueAndEvaluateInputExpression(this, lagRow,
inputColumnNum);
+ return batches.getValue(lagRow, inputColumnNum);
}
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java
index db0332ce9d9..d2806bb65b9 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java
@@ -38,7 +38,7 @@ public Object runOnRange(int rowNum, Range range,
VectorPTFGroupBatches batches)
if (leadRow < 0 || leadRow >= batches.size()) {
return getDefaultValue(rowNum, batches);
} else {
- return batches.getValueAndEvaluateInputExpression(this, leadRow,
inputColumnNum);
+ return batches.getValue(leadRow, inputColumnNum);
}
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java
index 9c5b402fa5c..5729a9a11e2 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java
@@ -46,7 +46,6 @@ public VectorPTFEvaluatorLongAvg(WindowFrameDef
windowFrameDef, VectorExpression
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null long column values for avg; maintain
isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java
index 1b1ca204939..d370a07880d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java
@@ -49,7 +49,6 @@ public VectorPTFEvaluatorLongFirstValue(WindowFrameDef
windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// First row determines isGroupResultNull and long firstValue; stream fill
result as repeated.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java
index bf0f27af832..536c05c9901 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorLongLastValue(WindowFrameDef
windowFrameDef,
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Last row of last batch determines isGroupResultNull and long lastValue.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java
index 47911af4521..fe6d3753839 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorLongMax(WindowFrameDef
windowFrameDef, VectorExpression
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null long column values; maintain
isGroupResultNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java
index 91d21f2f182..2867653d10b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorLongMin(WindowFrameDef
windowFrameDef, VectorExpression
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null long column values; maintain
isGroupResultNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java
index d37bf985dc1..5318407d6bb 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java
@@ -42,7 +42,6 @@ public VectorPTFEvaluatorLongSum(WindowFrameDef
windowFrameDef, VectorExpression
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null long column values; maintain isGroupResultNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java
index 3e4bc3d84bc..b287639c27a 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java
@@ -44,7 +44,6 @@ public VectorPTFEvaluatorRowNumber(WindowFrameDef
windowFrameDef, VectorExpressi
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
final int size = batch.size;
LongColumnVector longColVector = (LongColumnVector)
batch.cols[outputColumnNum];
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java
index e51d1fc5f66..58ad82003c5 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java
@@ -56,7 +56,6 @@ public VectorPTFEvaluatorStreamingDecimalAvg(WindowFrameDef
windowFrameDef, Vect
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null decimal column values for avg; maintain
isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java
index 93572427107..dabfa18b323 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorStreamingDecimalMax(WindowFrameDef
windowFrameDef, Vect
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null decimal column values; maintain
isNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java
index 51b43d7c622..a729775b00f 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorStreamingDecimalMin(WindowFrameDef
windowFrameDef, Vect
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null decimal column values; maintain
isNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java
index bc8620ac37f..3fb86f9379d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java
@@ -48,7 +48,6 @@ public VectorPTFEvaluatorStreamingDecimalSum(WindowFrameDef
windowFrameDef, Vect
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null decimal column values; maintain isGroupResultNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java
index f6c5942c065..18d521c07c2 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java
@@ -50,7 +50,6 @@ public VectorPTFEvaluatorStreamingDoubleAvg(WindowFrameDef
windowFrameDef, Vecto
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null double column values for avg; maintain
isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java
index 1d61cc5bf09..5a40b638e71 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingDoubleMax(WindowFrameDef
windowFrameDef, Vecto
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null double column values; maintain isNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java
index 9ac197d17a8..14c0b831edd 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingDoubleMin(WindowFrameDef
windowFrameDef, Vecto
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null double column values; maintain isNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java
index 8f17663a358..2e0dfa9a45f 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingDoubleSum(WindowFrameDef
windowFrameDef, Vecto
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// We do not filter when PTF is in reducer.
Preconditions.checkState(!batch.selectedInUse);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java
index 78d543a0c20..9117ac16d31 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java
@@ -50,7 +50,6 @@ public VectorPTFEvaluatorStreamingLongAvg(WindowFrameDef
windowFrameDef, VectorE
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null long column values for avg; maintain
isGroupResultNull; after last row of
// last group batch compute the group avg when sum is non-null.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java
index 94d19b3376f..ee2804b5838 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingLongMax(WindowFrameDef
windowFrameDef, VectorE
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine maximum of all non-null long column values; maintain isNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java
index 2d7caf34152..42ab267e947 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingLongMin(WindowFrameDef
windowFrameDef, VectorE
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Determine minimum of all non-null long column values; maintain isNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java
index 76bca6b6557..311707966c9 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java
@@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingLongSum(WindowFrameDef
windowFrameDef, VectorE
public void evaluateGroupBatch(VectorizedRowBatch batch)
throws HiveException {
- evaluateInputExpr(batch);
// Sum all non-null long column values; maintain isNull.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
index 8fee32a569c..a6547a11bcb 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java
@@ -346,15 +346,6 @@ public Object getValue(int row, int col) throws
HiveException {
return getValueFromBatch(bufferedBatches.get(rp.batchIndex), col,
rp.rowIndexInBatch);
}
- public Object getValueAndEvaluateInputExpression(VectorPTFEvaluatorBase
evaluator, int row,
- int col) throws HiveException {
- RowPositionInBatch rp = getPosition(row);
- BufferedVectorizedRowBatch batch = bufferedBatches.get(rp.batchIndex);
- if (!batch.isInputExpressionEvaluated) {
- evaluator.evaluateInputExpr(batch);
- }
- return getValueFromBatch(batch, col, rp.rowIndexInBatch);
- }
public RowPositionInBatch getPosition(int i) throws HiveException {
if (positionCache[i] != null) {
@@ -1012,7 +1003,6 @@ private VectorizedRowBatch
createTruncatedBufferedBatch(BufferedVectorizedRowBat
partialBatch.size = size;
partialBatch.isLastGroupBatch = bufferedBatch.isLastGroupBatch;
- partialBatch.isInputExpressionEvaluated =
bufferedBatch.isInputExpressionEvaluated;
return partialBatch;
}
@@ -1174,7 +1164,6 @@ private void readBlockFromSpillToMemory(VectorSpillBlock
block) throws HiveExcep
block.readSingleRowFromBytesContainer(bufferedBatch);
bufferedBatch.size += 1;
bufferedBatch.isLastGroupBatch = block.isLastGroupBatch[batchIndex];
- bufferedBatch.isInputExpressionEvaluated =
block.isInputExpressionEvaluated[batchIndex];
}
currentBufferedBatchCount = block.spillBatchCount;
partitionMetrics.batchesReadFromSpill += currentBufferedBatchCount;
@@ -1202,7 +1191,6 @@ private void spillAndResetCurrentBufferedBatches() throws
HiveException {
BufferedVectorizedRowBatch bufferedBatch = bufferedBatches.get(i);
block.spillBatch(bufferedBatch);
block.isLastGroupBatch[i] = bufferedBatch.isLastGroupBatch;
- block.isInputExpressionEvaluated[i] =
bufferedBatch.isInputExpressionEvaluated;
block.spillRowCount += bufferedBatch.size;
block.spillBatchCount += 1;
if (previousBatch != null) {
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
index 5accf6950c3..feb12617de1 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java
@@ -28,7 +28,6 @@
import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -390,6 +389,18 @@ public void process(Object row, int tag) throws
HiveException {
}
}
+ /*
+ Evaluate input expressions on the original batch before it gets buffered.
+ This ensures that complex expressions read from the correct columns in
+ the original batch and write their results to scratch columns, which
+ are then buffered.
+ */
+ for (VectorPTFEvaluatorBase evaluator : evaluators) {
+ if (evaluator.inputVecExpr != null) {
+ evaluator.inputVecExpr.evaluate(batch);
+ }
+ }
+
if (isFirstPartition) {
isFirstPartition = false;
setCurrentPartition(batch);
@@ -477,39 +488,27 @@ private void initBufferedColumns() {
}
private void initExpressionColumns() {
- for (int i = 0; i < evaluators.length; i++) {
- VectorPTFEvaluatorBase evaluator = evaluators[i];
+ for (VectorPTFEvaluatorBase evaluator : evaluators) {
/*
- * Non-streaming evaluators work on buffered batches, we need to adapt
them. Before PTF
- * bounded start vectorization (HIVE-24761),
VectorExpression.outputColumnNum was closed and
- * VectorExpression.inputColumnNum didn't even exist (even though the
vast majority of
- * VectorExpression subclasses use at least 1 input column). Since
VectorPTFOperator and
- * VectorPTFGroupBatches work on modified batches (by not storing all
the columns, and having
- * ordering columns first), the expressions planned in compile-time
won't work with the
- * original config (column layout). It would make sense to move this
logic to compile time,
- * because here in runtime, a very simple mapping (bufferedColumnMap) is
used, so it might be
- * used. However, vectorized expression compilation affects many layers
of code (having
- * VectorizationContext as the common scope), and moving the calculation
of bufferedColumnMap
- * and this override logic to compile-time would create much more
complicated behavior there
- * (probably involving hacking most of the time, or maybe a great
re-design) just because of
- * the optimized column layout of the PTF vectorization.
+ * Non-streaming window function evaluators (like SUM or LEAD) work on
packed, buffered batches.
+ * VectorPTFOperator and VectorPTFGroupBatches significantly modify the
batch structure to save memory
+ * (e.g. by dropping unused columns and moving ordering columns to the
front).
+ *
+ * Because of this dense packing, the absolute column indices assigned
by the compiler (e.g., Index 15)
+ * are no longer valid for the buffer (where it might now be Index 2).
+ * Therefore, we must dynamically map the evaluator's inputColumnNum to
its new location in the packed array.
+ *
+ * NOTE: We ONLY patch the evaluator's input index here. We specifically
DO NOT patch
+ * evaluator.inputVecExpr (the math expressions like A+B) because those
are now evaluated eagerly
+ * on the original batch before any packing occurs, so their absolute
indices remain perfectly valid.
*/
if (!evaluator.streamsResult()) {
- evaluator.inputColumnNum = IntStream.range(0, bufferedColumnMap.length)
- .filter(j -> bufferedColumnMap[j] ==
evaluator.inputColumnNum).findFirst()
- .orElseGet(() -> evaluator.inputColumnNum);
-
- if (evaluator.inputVecExpr != null) {
- for (int j = 0; j < evaluator.inputVecExpr.inputColumnNum.length;
j++){
- final int jj = j; // need a final in stream filter
- evaluator.inputVecExpr.inputColumnNum[jj] = IntStream.range(0,
bufferedColumnMap.length)
- .filter(k -> bufferedColumnMap[k] ==
evaluator.inputVecExpr.inputColumnNum[jj])
- .findFirst().orElseGet(() ->
evaluator.inputVecExpr.inputColumnNum[jj]);
- }
- evaluator.inputVecExpr.outputColumnNum = IntStream.range(0,
bufferedColumnMap.length)
- .filter(j -> bufferedColumnMap[j] ==
evaluator.inputVecExpr.outputColumnNum)
- .findFirst().orElseGet(() ->
evaluator.inputVecExpr.outputColumnNum);
- }
+ evaluator.inputColumnNum =
+ IntStream.range(0, bufferedColumnMap.length)
+ .filter(j -> bufferedColumnMap[j] == evaluator.inputColumnNum)
+ .findFirst()
+ .orElseGet(() -> evaluator.inputColumnNum);
+
evaluator.mapCustomColumns(bufferedColumnMap);
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java
index 8d21892b785..a5c6170324c 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java
@@ -139,10 +139,6 @@ class VectorSpillBlock {
*/
boolean[] isLastGroupBatch;
- /**
- * Stores the isInputExpressionEvaluated property for buffered/spilled
batches.
- */
- boolean[] isInputExpressionEvaluated;
/**
* <pre>
* Deserializer needs to know which columns to put the deserialized
values into. Basically,
@@ -165,7 +161,6 @@ class VectorSpillBlock {
startBatchIndex = blockIndex * blockSize;
startRowIndex = new int[blockSize];
isLastGroupBatch = new boolean[blockSize];
- isInputExpressionEvaluated = new boolean[blockSize];
}
void releaseRowBytesContainer() {
diff --git a/ql/src/test/queries/clientpositive/vector_ptf_nested_input_expr.q
b/ql/src/test/queries/clientpositive/vector_ptf_nested_input_expr.q
new file mode 100644
index 00000000000..823954565e9
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_ptf_nested_input_expr.q
@@ -0,0 +1,56 @@
+set hive.cli.print.header=true;
+set hive.vectorized.execution.enabled=true;
+set hive.vectorized.execution.reduce.enabled=true;
+set hive.vectorized.execution.ptf.enabled=true;
+
+CREATE TABLE repro_ptf_decimal (
+ id STRING,
+ val1 DECIMAL(20,2),
+ val2 BIGINT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB');
+
+INSERT INTO repro_ptf_decimal VALUES ('req1', 5.0, 2, 1), ('req1', 10.0, 4,
2), ('req2', 10.0, 5, 2);
+
+EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal;
+
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal;
+
+CREATE TABLE repro_ptf_long (
+ id STRING,
+ val1 BIGINT,
+ val2 INT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB');
+
+INSERT INTO repro_ptf_long VALUES ('req1', 5, 2, 1), ('req1', 10, 4, 2),
('req2', 10, 5, 2);
+
+EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long;
+
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long;
diff --git
a/ql/src/test/results/clientpositive/llap/vector_ptf_nested_input_expr.q.out
b/ql/src/test/results/clientpositive/llap/vector_ptf_nested_input_expr.q.out
new file mode 100644
index 00000000000..cf12f629649
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_ptf_nested_input_expr.q.out
@@ -0,0 +1,434 @@
+PREHOOK: query: CREATE TABLE repro_ptf_decimal (
+ id STRING,
+ val1 DECIMAL(20,2),
+ val2 BIGINT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repro_ptf_decimal
+POSTHOOK: query: CREATE TABLE repro_ptf_decimal (
+ id STRING,
+ val1 DECIMAL(20,2),
+ val2 BIGINT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repro_ptf_decimal
+PREHOOK: query: INSERT INTO repro_ptf_decimal VALUES ('req1', 5.0, 2, 1),
('req1', 10.0, 4, 2), ('req2', 10.0, 5, 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@repro_ptf_decimal
+POSTHOOK: query: INSERT INTO repro_ptf_decimal VALUES ('req1', 5.0, 2, 1),
('req1', 10.0, 4, 2), ('req2', 10.0, 5, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@repro_ptf_decimal
+POSTHOOK: Lineage: repro_ptf_decimal.id SCRIPT []
+POSTHOOK: Lineage: repro_ptf_decimal.val1 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_decimal.val2 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_decimal.val3 SCRIPT []
+_col0 _col1 _col2 _col3
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: repro_ptf_decimal
+ Statistics: Num rows: 3 Data size: 636 Basic stats: COMPLETE
Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:id:string,
1:val1:decimal(20,2), 2:val2:bigint, 3:val3:smallint,
4:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
5:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int), val1 (type:
decimal(20,2)), id (type: string)
+ null sort order: aaz
+ sort order: +-+
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 6:int, 1:decimal(20,2), 0:string
+ keyExpressions: ConstantVectorExpression(val 0) ->
6:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 7:int
+ valueColumns: 2:bigint, 3:smallint
+ Statistics: Num rows: 3 Data size: 636 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: val2 (type: bigint), val3 (type:
smallint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2, 3]
+ dataColumns: id:string, val1:decimal(20,2), val2:bigint,
val3:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: aaz
+ reduceColumnSortOrder: +-+
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ dataColumns: KEY.reducesinkkey0:int,
KEY.reducesinkkey1:decimal(20,2), KEY.reducesinkkey2:string,
VALUE._col0:bigint, VALUE._col1:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [decimal(34,2), bigint,
decimal(22,2), decimal(19,0), decimal(23,2), decimal(5,0), decimal(24,2)]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey2 (type: string),
KEY.reducesinkkey1 (type: decimal(20,2)), VALUE._col0 (type: bigint),
VALUE._col1 (type: smallint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4]
+ Statistics: Num rows: 3 Data size: 636 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: decimal(20,2),
_col2: bigint, _col3: smallint
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 DESC NULLS FIRST, _col0 ASC NULLS LAST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: (((_col1 * CAST( 2 AS decimal(10,0)))
+ CAST( _col2 AS decimal(19,0))) - CAST( _col3 AS decimal(5,0)))
+ name: sum
+ window function: GenericUDAFSumHiveDecimal
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorDecimalSum]
+ functionInputExpressions:
[DecimalColSubtractDecimalColumn(col 9:decimal(23,2), col
10:decimal(5,0))(children: DecimalColAddDecimalColumn(col 7:decimal(22,2), col
8:decimal(19,0))(children: DecimalColMultiplyDecimalScalar(col 1:decimal(20,2),
val 2) -> 7:decimal(22,2), CastLongToDecimal(col 3:bigint) -> 8:decimal(19,0))
-> 9:decimal(23,2), CastLongToDecimal(col 4:smallint) -> 10:decimal(5,0)) ->
11:decimal(24,2)]
+ functionNames: [sum]
+ keyInputColumns: [2, 1]
+ native: true
+ nonKeyInputColumns: [3, 4]
+ orderExpressions: [col 1:decimal(20,2), col 2:string]
+ outputColumns: [5, 2, 1, 3, 4]
+ outputTypes: [decimal(34,2), string, decimal(20,2),
bigint, smallint]
+ partitionExpressions: [ConstantVectorExpression(val 0)
-> 6:int]
+ streamingColumns: []
+ Statistics: Num rows: 3 Data size: 636 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type:
decimal(20,2)), _col2 (type: bigint), _col3 (type: smallint), sum_window_0
(type: decimal(34,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4, 5]
+ Statistics: Num rows: 3 Data size: 972 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 3 Data size: 972 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+id val1 val2 val3 final_total
+req1 10.00 4 2 22.00
+req2 10.00 5 2 45.00
+req1 5.00 2 1 56.00
+PREHOOK: query: CREATE TABLE repro_ptf_long (
+ id STRING,
+ val1 BIGINT,
+ val2 INT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repro_ptf_long
+POSTHOOK: query: CREATE TABLE repro_ptf_long (
+ id STRING,
+ val1 BIGINT,
+ val2 INT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repro_ptf_long
+PREHOOK: query: INSERT INTO repro_ptf_long VALUES ('req1', 5, 2, 1), ('req1',
10, 4, 2), ('req2', 10, 5, 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@repro_ptf_long
+POSTHOOK: query: INSERT INTO repro_ptf_long VALUES ('req1', 5, 2, 1), ('req1',
10, 4, 2), ('req2', 10, 5, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@repro_ptf_long
+POSTHOOK: Lineage: repro_ptf_long.id SCRIPT []
+POSTHOOK: Lineage: repro_ptf_long.val1 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_long.val2 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_long.val3 SCRIPT []
+_col0 _col1 _col2 _col3
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: repro_ptf_long
+ Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE
Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:id:string, 1:val1:bigint,
2:val2:int, 3:val3:smallint,
4:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
5:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int), val1 (type: bigint), id
(type: string)
+ null sort order: aaz
+ sort order: +-+
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 6:int, 1:bigint, 0:string
+ keyExpressions: ConstantVectorExpression(val 0) ->
6:int
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 7:int
+ valueColumns: 2:int, 3:smallint
+ Statistics: Num rows: 3 Data size: 312 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: val2 (type: int), val3 (type: smallint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2, 3]
+ dataColumns: id:string, val1:bigint, val2:int,
val3:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: aaz
+ reduceColumnSortOrder: +-+
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ dataColumns: KEY.reducesinkkey0:int,
KEY.reducesinkkey1:bigint, KEY.reducesinkkey2:string, VALUE._col0:int,
VALUE._col1:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, bigint, bigint,
bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey2 (type: string),
KEY.reducesinkkey1 (type: bigint), VALUE._col0 (type: int), VALUE._col1 (type:
smallint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4]
+ Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE
Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: bigint, _col2:
int, _col3: smallint
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 DESC NULLS FIRST, _col0 ASC NULLS LAST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: (((_col1 * UDFToLong(2)) +
UDFToLong(_col2)) - UDFToLong(_col3))
+ name: sum
+ window function: GenericUDAFSumLong
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorLongSum]
+ functionInputExpressions: [LongColSubtractLongColumn(col
8:bigint, col 4:bigint)(children: LongColAddLongColumn(col 7:bigint, col
3:bigint)(children: LongColMultiplyLongScalar(col 1:bigint, val 2) -> 7:bigint,
col 3:int) -> 8:bigint, col 4:smallint) -> 9:bigint]
+ functionNames: [sum]
+ keyInputColumns: [2, 1]
+ native: true
+ nonKeyInputColumns: [3, 4]
+ orderExpressions: [col 1:bigint, col 2:string]
+ outputColumns: [5, 2, 1, 3, 4]
+ outputTypes: [bigint, string, bigint, int, smallint]
+ partitionExpressions: [ConstantVectorExpression(val 0)
-> 6:int]
+ streamingColumns: []
+ Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint),
_col2 (type: int), _col3 (type: smallint), sum_window_0 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4, 5]
+ Statistics: Num rows: 3 Data size: 336 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 3 Data size: 336 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+id val1 val2 val3 final_total
+req1 10 4 2 22
+req2 10 5 2 45
+req1 5 2 1 56
