This is an automated email from the ASF dual-hosted git repository.
kasakrisz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c018fab39b4 HIVE-28329: Vectorized multiplication of CASE expression
and decimal column incorrectly returns 0 (#6575)
c018fab39b4 is described below
commit c018fab39b4015f953edd9e5c68769add5238fc2
Author: kokila-19 <[email protected]>
AuthorDate: Fri Jul 24 13:43:38 2026 +0530
HIVE-28329: Vectorized multiplication of CASE expression and decimal column
incorrectly returns 0 (#6575)
Preserve inputColumnNum column refs when re-instantiating DECIMAL parents
after DECIMAL_64 child conversion and refresh projectedOutputColumns.
---
.../hive/ql/optimizer/physical/Vectorizer.java | 104 +++++++--
...tor_decimal64_col_multiply_case_subquery_join.q | 21 ++
...decimal64_col_multiply_case_subquery_join.q.out | 250 +++++++++++++++++++++
3 files changed, 358 insertions(+), 17 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 82f81861a4d..143d7de77b6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -102,6 +102,7 @@
import
org.apache.hadoop.hive.ql.exec.vector.VectorizationContext.InConstantType;
import org.apache.hadoop.hive.ql.exec.vector.VectorizationContextRegion;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedSupport.Support;
+import
org.apache.hadoop.hive.ql.exec.vector.expressions.ConvertDecimal64ToDecimal;
import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
@@ -4796,15 +4797,19 @@ public static Operator<? extends OperatorDesc>
vectorizeSelectOperator(
vContext.markActualScratchColumns();
VectorExpression[] vectorSelectExprs = new VectorExpression[size];
+ VectorExpression[] selectColumnExprs = new VectorExpression[size];
int[] projectedOutputColumns = new int[size];
+ int[] vectorSelectExprIndexForCol = new int[size];
+ Arrays.fill(vectorSelectExprIndexForCol, -1);
for (int i = 0; i < size; i++) {
ExprNodeDesc expr = colList.get(i);
VectorExpression ve = vContext.getVectorExpression(expr);
- projectedOutputColumns[i] = ve.getOutputColumnNum();
+ selectColumnExprs[i] = ve;
if (ve instanceof IdentityExpression) {
// Suppress useless evaluation.
continue;
}
+ vectorSelectExprIndexForCol[i] = index;
vectorSelectExprs[index++] = ve;
}
if (index < size) {
@@ -4817,6 +4822,13 @@ public static Operator<? extends OperatorDesc>
vectorizeSelectOperator(
// The following method introduces a cast if x or y is DECIMAL_64 and
parent expression (x % y) is DECIMAL.
try {
fixDecimalDataTypePhysicalVariations(vContext, vectorSelectExprs);
+ for (int i = 0; i < size; i++) {
+ int exprIndex = vectorSelectExprIndexForCol[i];
+ VectorExpression ve = (exprIndex >= 0)
+ ? vectorSelectExprs[exprIndex]
+ : selectColumnExprs[i];
+ projectedOutputColumns[i] = ve.getOutputColumnNum();
+ }
} finally {
vContext.freeMarkedScratchColumns();
}
@@ -4879,7 +4891,6 @@ private static VectorExpression
fixDecimalDataTypePhysicalVariations(final Vecto
}
} else {
Object[] arguments;
- int argumentCount = children.length + (parent.getOutputColumnNum()
== -1 ? 0 : 1);
// VectorCoalesce receives arguments as an array.
// Need to handle it as a special case to avoid instantiation
failure.
if (parent instanceof VectorCoalesce) {
@@ -4891,20 +4902,7 @@ private static VectorExpression
fixDecimalDataTypePhysicalVariations(final Vecto
}
arguments[1] = parent.getOutputColumnNum();
} else {
- if (parent instanceof DecimalColDivideDecimalScalar) {
- arguments = new Object[argumentCount + 1];
- arguments[children.length] = ((DecimalColDivideDecimalScalar)
parent).getValue();
- } else {
- arguments = new Object[argumentCount];
- }
- for (int i = 0; i < children.length; i++) {
- VectorExpression vce = children[i];
- arguments[i] = vce.getOutputColumnNum();
- }
- }
- // retain output column number from parent
- if (parent.getOutputColumnNum() != -1) {
- arguments[arguments.length - 1] = parent.getOutputColumnNum();
+ arguments = buildReinstantiationArgsForDecimal64(parent, children);
}
// re-instantiate the parent expression with new arguments
VectorExpression newParent =
vContext.instantiateExpression(parent.getClass(), parent.getOutputTypeInfo(),
@@ -4913,7 +4911,7 @@ private static VectorExpression
fixDecimalDataTypePhysicalVariations(final Vecto
newParent.setOutputDataTypePhysicalVariation(parent.getOutputDataTypePhysicalVariation());
newParent.setInputTypeInfos(parent.getInputTypeInfos());
newParent.setInputDataTypePhysicalVariations(dataTypePhysicalVariations);
- newParent.setChildExpressions(parent.getChildExpressions());
+ newParent.setChildExpressions(children);
return newParent;
}
}
@@ -4921,6 +4919,78 @@ private static VectorExpression
fixDecimalDataTypePhysicalVariations(final Vecto
return parent;
}
+ /**
+ * Rebuild constructor arguments for a vector expression after wrapping
DECIMAL_64 child
+ * expressions with {@link ConvertDecimal64ToDecimal}. Column reference
inputs live in
+ * {@link VectorExpression#inputColumnNum} and are not included in
childExpressions, so they
+ * must be preserved when re-instantiating the parent.
+ */
+ static Object[] buildReinstantiationArgsForDecimal64(VectorExpression parent,
+ VectorExpression[] children) {
+ int[] inputColNums = extractParentInputColumnNums(parent);
+ replaceInputColsWithConvertedChildOutputs(inputColNums, children);
+ return buildParentConstructorArguments(inputColNums, parent);
+ }
+
+ private static int[] extractParentInputColumnNums(VectorExpression parent) {
+ int inputCount = 0;
+ for (int col : parent.inputColumnNum) {
+ if (col != -1) {
+ inputCount++;
+ }
+ }
+
+ int[] inputColNums = new int[inputCount];
+ int idx = 0;
+ for (int col : parent.inputColumnNum) {
+ if (col != -1) {
+ inputColNums[idx++] = col;
+ }
+ }
+ return inputColNums;
+ }
+
+ /**
+ * For each wrapped DECIMAL_64 child, replace its pre-conversion input
column slot in
+ * {@code inputColNums} with the {@link ConvertDecimal64ToDecimal} output
column.
+ */
+ private static void replaceInputColsWithConvertedChildOutputs(int[]
inputColNums,
+ VectorExpression[] children) {
+ for (VectorExpression child : children) {
+ int preConversionCol = getPreConversionColumnNum(child);
+ int convertedCol = child.getOutputColumnNum();
+ for (int i = 0; i < inputColNums.length; i++) {
+ if (inputColNums[i] == preConversionCol) {
+ inputColNums[i] = convertedCol;
+ break;
+ }
+ }
+ }
+ }
+
+ private static Object[] buildParentConstructorArguments(int[] inputColNums,
+ VectorExpression parent) {
+ int extraArgs = parent instanceof DecimalColDivideDecimalScalar ? 2 : 1;
+ Object[] arguments = new Object[inputColNums.length + extraArgs];
+ for (int i = 0; i < inputColNums.length; i++) {
+ arguments[i] = inputColNums[i];
+ }
+ int outputIndex = inputColNums.length;
+ if (parent instanceof DecimalColDivideDecimalScalar) {
+ arguments[outputIndex++] = ((DecimalColDivideDecimalScalar)
parent).getValue();
+ }
+ arguments[outputIndex] = parent.getOutputColumnNum();
+ return arguments;
+ }
+
+ /** Column to match in {@code inputColNums} before replacing with a
converted child output. */
+ private static int getPreConversionColumnNum(VectorExpression child) {
+ if (child instanceof ConvertDecimal64ToDecimal) {
+ return child.inputColumnNum[0];
+ }
+ return child.getOutputColumnNum();
+ }
+
private static void fillInPTFEvaluators(
List<WindowFunctionDef> windowsFunctions,
String[] evaluatorFunctionNames,
diff --git
a/ql/src/test/queries/clientpositive/vector_decimal64_col_multiply_case_subquery_join.q
b/ql/src/test/queries/clientpositive/vector_decimal64_col_multiply_case_subquery_join.q
new file mode 100644
index 00000000000..ed4cb9365ba
--- /dev/null
+++
b/ql/src/test/queries/clientpositive/vector_decimal64_col_multiply_case_subquery_join.q
@@ -0,0 +1,21 @@
+CREATE TABLE test1(col2 varchar(28), col3 decimal(26,6), col4 varchar(28));
+CREATE TABLE test2(col4 varchar(28));
+
+INSERT INTO test1 VALUES ('abc', 1.00, 'DEF');
+INSERT INTO test2 VALUES ('DEF');
+
+EXPLAIN VECTORIZATION DETAIL
+SELECT
+ aa.col3 * CASE WHEN aa.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost
+FROM test1 aa
+ INNER JOIN
+ test2 bb ON aa.col4 = bb.col4;
+
+SELECT
+ aa.col3 * CASE WHEN aa.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost
+FROM test1 aa
+ INNER JOIN
+ test2 bb ON aa.col4 = bb.col4;
+
+DROP TABLE test1;
+DROP TABLE test2;
\ No newline at end of file
diff --git
a/ql/src/test/results/clientpositive/llap/vector_decimal64_col_multiply_case_subquery_join.q.out
b/ql/src/test/results/clientpositive/llap/vector_decimal64_col_multiply_case_subquery_join.q.out
new file mode 100644
index 00000000000..e260248c4a1
--- /dev/null
+++
b/ql/src/test/results/clientpositive/llap/vector_decimal64_col_multiply_case_subquery_join.q.out
@@ -0,0 +1,250 @@
+PREHOOK: query: CREATE TABLE test1(col2 varchar(28), col3 decimal(26,6), col4
varchar(28))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test1
+POSTHOOK: query: CREATE TABLE test1(col2 varchar(28), col3 decimal(26,6), col4
varchar(28))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test1
+PREHOOK: query: CREATE TABLE test2(col4 varchar(28))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test2
+POSTHOOK: query: CREATE TABLE test2(col4 varchar(28))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test2
+PREHOOK: query: INSERT INTO test1 VALUES ('abc', 1.00, 'DEF')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test1
+POSTHOOK: query: INSERT INTO test1 VALUES ('abc', 1.00, 'DEF')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test1
+POSTHOOK: Lineage: test1.col2 SCRIPT []
+POSTHOOK: Lineage: test1.col3 SCRIPT []
+POSTHOOK: Lineage: test1.col4 SCRIPT []
+PREHOOK: query: INSERT INTO test2 VALUES ('DEF')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test2
+POSTHOOK: query: INSERT INTO test2 VALUES ('DEF')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test2
+POSTHOOK: Lineage: test2.col4 SCRIPT []
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ aa.col3 * CASE WHEN aa.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost
+FROM test1 aa
+ INNER JOIN
+ test2 bb ON aa.col4 = bb.col4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+PREHOOK: Input: default@test2
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ aa.col3 * CASE WHEN aa.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost
+FROM test1 aa
+ INNER JOIN
+ test2 bb ON aa.col4 = bb.col4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test1
+POSTHOOK: Input: default@test2
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: aa
+ filterExpr: col4 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 286 Basic stats: COMPLETE
Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:col2:varchar(28),
1:col3:decimal(26,6), 2:col4:varchar(28),
3:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
4:ROW__IS__DELETED:boolean]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: SelectColumnIsNotNull(col
2:varchar(28))
+ predicate: col4 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 286 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: col4 (type: varchar(28)), (col3 * if((col2
= 'bc'), 1.77, 0.72)) (type: decimal(30,8))
+ outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 7]
+ selectExpressions:
DecimalColMultiplyDecimalColumn(col 1:decimal(26,6), col
8:decimal(3,2)/DECIMAL_64)(children: ConvertDecimal64ToDecimal(col
6:decimal(3,2)/DECIMAL_64)(children: IfExprDecimal64ScalarDecimal64Scalar(col
5:boolean, decimal64Val1 177, decimalVal1 1.77, decimal64Val2 72, decimalVal2
0.72)(children: StringGroupColEqualVarCharScalar(col 0:varchar(28), val bc) ->
5:boolean) -> 6:decimal(3,2)/DECIMAL_64) -> 8:decimal(3,2)) -> 7:decimal(30,8)
+ Statistics: Num rows: 1 Data size: 199 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: varchar(28))
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: varchar(28))
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 2:varchar(28)
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 7:decimal(30,8)
+ Statistics: Num rows: 1 Data size: 199 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: decimal(30,8))
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ includeColumns: [0, 1, 2]
+ dataColumns: col2:varchar(28), col3:decimal(26,6),
col4:varchar(28)
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, decimal(3,2)/DECIMAL_64,
decimal(30,8), decimal(3,2)]
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: bb
+ filterExpr: col4 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE
Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:col4:varchar(28),
1:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>,
2:ROW__IS__DELETED:boolean]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: SelectColumnIsNotNull(col
0:varchar(28))
+ predicate: col4 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 87 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: col4 (type: varchar(28))
+ outputColumnNames: _col0
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0]
+ Statistics: Num rows: 1 Data size: 87 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: varchar(28))
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: varchar(28))
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:varchar(28)
+ native: true
+ nativeConditionsMet:
hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine
tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true,
BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ Statistics: Num rows: 1 Data size: 87 Basic stats:
COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vector.serde.deserialize IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 1
+ includeColumns: [0]
+ dataColumns: col4:varchar(28)
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: varchar(28))
+ 1 _col0 (type: varchar(28))
+ outputColumnNames: _col1
+ Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: _col1 (type: decimal(30,8))
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 112 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ MergeJoin Vectorization:
+ enabled: false
+ enableConditionsNotMet: Vectorizing MergeJoin Supported IS
false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT
+ aa.col3 * CASE WHEN aa.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost
+FROM test1 aa
+ INNER JOIN
+ test2 bb ON aa.col4 = bb.col4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test1
+PREHOOK: Input: default@test2
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ aa.col3 * CASE WHEN aa.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost
+FROM test1 aa
+ INNER JOIN
+ test2 bb ON aa.col4 = bb.col4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test1
+POSTHOOK: Input: default@test2
+#### A masked pattern was here ####
+0.72000000
+PREHOOK: query: DROP TABLE test1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@test1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test1
+POSTHOOK: query: DROP TABLE test1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@test1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test1
+PREHOOK: query: DROP TABLE test2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@test2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test2
+POSTHOOK: query: DROP TABLE test2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@test2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test2
