This is an automated email from the ASF dual-hosted git repository.
sbadhya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5b76949da6f HIVE-27950: STACK UDTF returns wrong results when number
of arguments is not a multiple of N (#4938) (okumin reviewed by Attila Turoczy,
Zsolt Miskolczi and Sourabh Badhya)
5b76949da6f is described below
commit 5b76949da6fe65364a4e3766680871167131157f
Author: okumin <[email protected]>
AuthorDate: Tue Feb 20 14:17:43 2024 +0900
HIVE-27950: STACK UDTF returns wrong results when number of arguments is
not a multiple of N (#4938) (okumin reviewed by Attila Turoczy, Zsolt Miskolczi
and Sourabh Badhya)
---
.../hive/ql/udf/generic/GenericUDTFStack.java | 20 +++++++----
.../clientnegative/udtf_stack_not_constant.q | 2 ++
.../test/queries/clientnegative/udtf_stack_null.q | 1 +
.../clientnegative/udtf_stack_wrong_type1.q | 1 +
ql/src/test/queries/clientpositive/udtf_stack.q | 3 ++
.../clientnegative/udtf_stack_not_constant.q.out | 1 +
.../results/clientnegative/udtf_stack_null.q.out | 1 +
.../clientnegative/udtf_stack_wrong_type1.q.out | 1 +
.../clientpositive/llap/allcolref_in_udf.q.out | 10 +++---
.../results/clientpositive/llap/udtf_stack.q.out | 39 ++++++++++++++++++++++
10 files changed, 68 insertions(+), 11 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java
index f3cc2b4f638..d75d3b62714 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java
@@ -23,13 +23,13 @@ import java.util.ArrayList;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import
org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils.ReturnObjectInspectorResolver;
import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import
org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantIntObjectInspector;
import org.apache.hadoop.io.IntWritable;
/**
@@ -63,13 +63,21 @@ public class GenericUDTFStack extends GenericUDTF {
}
if (!(args[0] instanceof ConstantObjectInspector)) {
throw new UDFArgumentException(
+ "The first argument to STACK() must be a constant.");
+ }
+ final Object value = ((ConstantObjectInspector)
args[0]).getWritableConstantValue();
+ if (value == null) {
+ throw new UDFArgumentException("The first argument of STACK() must not
be null.");
+ }
+ if (!(value instanceof IntWritable)) {
+ throw new UDFArgumentTypeException(
+ 0,
"The first argument to STACK() must be a constant integer (got " +
args[0].getTypeName() + " instead).");
}
- numRows = (IntWritable)
- ((ConstantObjectInspector)args[0]).getWritableConstantValue();
+ numRows = (IntWritable) value;
- if (numRows == null || numRows.get() < 1) {
+ if (numRows.get() < 1) {
throw new UDFArgumentException(
"STACK() expects its first argument to be >= 1.");
}
@@ -109,7 +117,7 @@ public class GenericUDTFStack extends GenericUDTF {
@Override
public void process(Object[] args)
- throws HiveException, UDFArgumentException {
+ throws HiveException {
for (int ii = 0; ii < numRows.get(); ++ii) {
for (int jj = 0; jj < numCols; ++jj) {
int index = ii * numCols + jj + 1;
@@ -117,7 +125,7 @@ public class GenericUDTFStack extends GenericUDTF {
forwardObj[jj] =
returnOIResolvers.get(jj).convertIfNecessary(args[index],
argOIs.get(index));
} else {
- forwardObj[ii] = null;
+ forwardObj[jj] = null;
}
}
forward(forwardObj);
diff --git a/ql/src/test/queries/clientnegative/udtf_stack_not_constant.q
b/ql/src/test/queries/clientnegative/udtf_stack_not_constant.q
new file mode 100644
index 00000000000..f559ce30462
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/udtf_stack_not_constant.q
@@ -0,0 +1,2 @@
+--! qt:dataset:alltypesparquet
+SELECT STACK(cint, 'a', 'b') FROM alltypesparquet;
diff --git a/ql/src/test/queries/clientnegative/udtf_stack_null.q
b/ql/src/test/queries/clientnegative/udtf_stack_null.q
new file mode 100644
index 00000000000..f3832ffd4ad
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/udtf_stack_null.q
@@ -0,0 +1 @@
+SELECT stack(cast(null as int), 'a', 'b', 'c', 'd');
diff --git a/ql/src/test/queries/clientnegative/udtf_stack_wrong_type1.q
b/ql/src/test/queries/clientnegative/udtf_stack_wrong_type1.q
new file mode 100644
index 00000000000..90762becb23
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/udtf_stack_wrong_type1.q
@@ -0,0 +1 @@
+SELECT stack('2', 'a', 'b', 'c', 'd');
diff --git a/ql/src/test/queries/clientpositive/udtf_stack.q
b/ql/src/test/queries/clientpositive/udtf_stack.q
index 8aa7a8cfe80..def0c38f5d5 100644
--- a/ql/src/test/queries/clientpositive/udtf_stack.q
+++ b/ql/src/test/queries/clientpositive/udtf_stack.q
@@ -11,3 +11,6 @@ SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1),
'z', array(4)) a AS x,
EXPLAIN
SELECT stack(1, "en", "dbpedia", NULL );
SELECT stack(1, "en", "dbpedia", NULL );
+
+EXPLAIN SELECT STACK(2, 'a', 'b', 'c', 'd', 'e');
+SELECT STACK(2, 'a', 'b', 'c', 'd', 'e');
diff --git a/ql/src/test/results/clientnegative/udtf_stack_not_constant.q.out
b/ql/src/test/results/clientnegative/udtf_stack_not_constant.q.out
new file mode 100644
index 00000000000..86b264a179d
--- /dev/null
+++ b/ql/src/test/results/clientnegative/udtf_stack_not_constant.q.out
@@ -0,0 +1 @@
+FAILED: UDFArgumentException The first argument to STACK() must be a constant.
diff --git a/ql/src/test/results/clientnegative/udtf_stack_null.q.out
b/ql/src/test/results/clientnegative/udtf_stack_null.q.out
new file mode 100644
index 00000000000..854a2361c3a
--- /dev/null
+++ b/ql/src/test/results/clientnegative/udtf_stack_null.q.out
@@ -0,0 +1 @@
+FAILED: UDFArgumentException The first argument of STACK() must not be null.
diff --git a/ql/src/test/results/clientnegative/udtf_stack_wrong_type1.q.out
b/ql/src/test/results/clientnegative/udtf_stack_wrong_type1.q.out
new file mode 100644
index 00000000000..d18a726b9c1
--- /dev/null
+++ b/ql/src/test/results/clientnegative/udtf_stack_wrong_type1.q.out
@@ -0,0 +1 @@
+FAILED: UDFArgumentTypeException The first argument to STACK() must be a
constant integer (got string instead).
diff --git a/ql/src/test/results/clientpositive/llap/allcolref_in_udf.q.out
b/ql/src/test/results/clientpositive/llap/allcolref_in_udf.q.out
index 8e20aaa35d8..236d52a2ded 100644
--- a/ql/src/test/results/clientpositive/llap/allcolref_in_udf.q.out
+++ b/ql/src/test/results/clientpositive/llap/allcolref_in_udf.q.out
@@ -168,15 +168,15 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
4val_45val_5 4val_4 5val_5
-4val_45 NULL 5val_5
+4val_45 45val_5 NULL
4val_45val_5 4val_4 5val_5
-4val_45 NULL 5val_5
+4val_45 45val_5 NULL
4val_45val_5 4val_4 5val_5
-4val_45 NULL 5val_5
+4val_45 45val_5 NULL
8val_89val_9 8val_8 9val_9
-8val_89 NULL 9val_9
+8val_89 89val_9 NULL
9val_910val_10 9val_9 10val_10
-9val_910 NULL 10val_10
+9val_910 910val_10 NULL
PREHOOK: query: create table allcolref as select array(key, value) from src
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
diff --git a/ql/src/test/results/clientpositive/llap/udtf_stack.q.out
b/ql/src/test/results/clientpositive/llap/udtf_stack.q.out
index 2e5bba2e388..ad176cea611 100644
--- a/ql/src/test/results/clientpositive/llap/udtf_stack.q.out
+++ b/ql/src/test/results/clientpositive/llap/udtf_stack.q.out
@@ -147,3 +147,42 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
#### A masked pattern was here ####
en dbpedia NULL
+PREHOOK: query: EXPLAIN SELECT STACK(2, 'a', 'b', 'c', 'd', 'e')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN SELECT STACK(2, 'a', 'b', 'c', 'd', 'e')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Select Operator
+ expressions: 2 (type: int), 'a' (type: string), 'b' (type:
string), 'c' (type: string), 'd' (type: string), 'e' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ UDTF Operator
+ function name: stack
+ Select Operator
+ expressions: col0 (type: string), col1 (type: string), col2
(type: string)
+ outputColumnNames: _col0, _col1, _col2
+ ListSink
+
+PREHOOK: query: SELECT STACK(2, 'a', 'b', 'c', 'd', 'e')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT STACK(2, 'a', 'b', 'c', 'd', 'e')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+a b c
+d e NULL
