This is an automated email from the ASF dual-hosted git repository.
ueshin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b81da764b9c5 [SPARK-49995][SQL] Add named argument support to more TVFs
b81da764b9c5 is described below
commit b81da764b9c562efeb820e8d3850a8e7a60c7bf7
Author: Takuya Ueshin <[email protected]>
AuthorDate: Tue Nov 5 12:45:28 2024 -0800
[SPARK-49995][SQL] Add named argument support to more TVFs
### What changes were proposed in this pull request?
Add named argument support to more TVFs.
```sql
SELECT * FROM inline(input => array(struct(1, 'a'), struct(2, 'b')));
SELECT * FROM inline_outer(input => array(struct(1, 'a'), struct(2, 'b')));
SELECT * FROM posexplode(collection => array(1, 2));
SELECT * FROM posexplode_outer(collection => map('a', 1, 'b', 2));
SELECT * FROM variant_explode(input => parse_json('["hello", "world"]'));
SELECT * FROM variant_explode_outer(input => parse_json('{"a": true, "b":
3.14}'));
```
### Why are the changes needed?
The following TVFs should support named argument as same as `explode`:
- `inline`
- `posexplode`
- `variant_explode`
and their `_outer` variations.
### Does this PR introduce _any_ user-facing change?
Yes, the above functions support named arguments.
### How was this patch tested?
Added the related tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #48503 from ueshin/issues/SPARK-49995/named_argument.
Authored-by: Takuya Ueshin <[email protected]>
Signed-off-by: Takuya Ueshin <[email protected]>
---
.../sql/catalyst/analysis/FunctionRegistry.scala | 20 +--
.../sql/catalyst/expressions/generators.scala | 156 +++++++++++++++++----
.../expressions/variant/variantExpressions.scala | 66 ++++++---
.../named-function-arguments.sql.out | 98 +++++++++++++
.../variant/named-function-arguments.sql.out | 48 +++++++
.../sql-tests/inputs/named-function-arguments.sql | 14 ++
.../inputs/variant/named-function-arguments.sql | 6 +
.../results/named-function-arguments.sql.out | 96 +++++++++++++
.../variant/named-function-arguments.sql.out | 48 +++++++
.../thriftserver/ThriftServerQueryTestSuite.scala | 4 +-
10 files changed, 503 insertions(+), 53 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 4ad0b81b8f26..8fe5c1b59902 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -367,8 +367,8 @@ object FunctionRegistry {
expressionGeneratorBuilderOuter("explode_outer", ExplodeExpressionBuilder),
expression[Greatest]("greatest"),
expression[If]("if"),
- expression[Inline]("inline"),
- expressionGeneratorOuter[Inline]("inline_outer"),
+ expressionBuilder("inline", InlineExpressionBuilder),
+ expressionGeneratorBuilderOuter("inline_outer", InlineExpressionBuilder),
expression[IsNaN]("isnan"),
expression[Nvl]("ifnull", setAlias = true),
expression[IsNull]("isnull"),
@@ -379,8 +379,8 @@ object FunctionRegistry {
expression[NullIfZero]("nullifzero"),
expression[Nvl]("nvl"),
expression[Nvl2]("nvl2"),
- expression[PosExplode]("posexplode"),
- expressionGeneratorOuter[PosExplode]("posexplode_outer"),
+ expressionBuilder("posexplode", PosExplodeExpressionBuilder),
+ expressionGeneratorBuilderOuter("posexplode_outer",
PosExplodeExpressionBuilder),
expression[Rand]("rand"),
expression[Rand]("random", true, Some("3.0.0")),
expression[Randn]("randn"),
@@ -1172,16 +1172,16 @@ object TableFunctionRegistry {
logicalPlan[Range]("range"),
generatorBuilder("explode", ExplodeGeneratorBuilder),
generatorBuilder("explode_outer", ExplodeOuterGeneratorBuilder),
- generator[Inline]("inline"),
- generator[Inline]("inline_outer", outer = true),
+ generatorBuilder("inline", InlineGeneratorBuilder),
+ generatorBuilder("inline_outer", InlineOuterGeneratorBuilder),
generator[JsonTuple]("json_tuple"),
- generator[PosExplode]("posexplode"),
- generator[PosExplode]("posexplode_outer", outer = true),
+ generatorBuilder("posexplode", PosExplodeGeneratorBuilder),
+ generatorBuilder("posexplode_outer", PosExplodeOuterGeneratorBuilder),
generator[Stack]("stack"),
generator[Collations]("collations"),
generator[SQLKeywords]("sql_keywords"),
- generator[VariantExplode]("variant_explode"),
- generator[VariantExplode]("variant_explode_outer", outer = true)
+ generatorBuilder("variant_explode", VariantExplodeGeneratorBuilder),
+ generatorBuilder("variant_explode_outer",
VariantExplodeOuterGeneratorBuilder)
)
val builtin: SimpleTableFunctionRegistry = {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index dc58352a1b36..b513b3858bbd 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -445,9 +445,6 @@ trait ExplodeGeneratorBuilderBase extends GeneratorBuilder {
> SELECT _FUNC_(collection => array(10, 20));
10
20
- > SELECT * FROM _FUNC_(collection => array(10, 20));
- 10
- 20
""",
since = "1.0.0",
group = "generator_funcs")
@@ -465,17 +462,14 @@ object ExplodeExpressionBuilder extends ExpressionBuilder
{
usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple
rows, or the elements of map `expr` into multiple rows and columns. Unless
specified otherwise, uses the default column name `col` for elements of the
array or `key` and `value` for the elements of the map.",
examples = """
Examples:
- > SELECT _FUNC_(array(10, 20));
- 10
- 20
- > SELECT _FUNC_(collection => array(10, 20));
+ > SELECT * FROM _FUNC_(array(10, 20));
10
20
> SELECT * FROM _FUNC_(collection => array(10, 20));
10
20
""",
- since = "1.0.0",
+ since = "3.4.0",
group = "generator_funcs")
// scalastyle:on line.size.limit
object ExplodeGeneratorBuilder extends ExplodeGeneratorBuilderBase {
@@ -487,21 +481,20 @@ object ExplodeGeneratorBuilder extends
ExplodeGeneratorBuilderBase {
usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple
rows, or the elements of map `expr` into multiple rows and columns. Unless
specified otherwise, uses the default column name `col` for elements of the
array or `key` and `value` for the elements of the map.",
examples = """
Examples:
- > SELECT _FUNC_(array(10, 20));
+ > SELECT * FROM _FUNC_(array(10, 20));
10
20
- > SELECT _FUNC_(collection => array(10, 20));
+ > SELECT * FROM _FUNC_(collection => array(10, 20));
10
20
""",
- since = "1.0.0",
+ since = "3.4.0",
group = "generator_funcs")
// scalastyle:on line.size.limit
object ExplodeOuterGeneratorBuilder extends ExplodeGeneratorBuilderBase {
override def isOuter: Boolean = true
}
-
/**
* Given an input array produces a sequence of rows for each position and
value in the array.
*
@@ -511,6 +504,21 @@ object ExplodeOuterGeneratorBuilder extends
ExplodeGeneratorBuilderBase {
* 1 20
* }}}
*/
+case class PosExplode(child: Expression) extends ExplodeBase {
+ override val position = true
+ override protected def withNewChildInternal(newChild: Expression):
PosExplode =
+ copy(child = newChild)
+}
+
+trait PosExplodeGeneratorBuilderBase extends GeneratorBuilder {
+ override def functionSignature: Option[FunctionSignature] =
+ Some(FunctionSignature(Seq(InputParameter("collection"))))
+ override def buildGenerator(funcName: String, expressions: Seq[Expression]):
Generator = {
+ assert(expressions.size == 1)
+ PosExplode(expressions(0))
+ }
+}
+
// scalastyle:off line.size.limit line.contains.tab
@ExpressionDescription(
usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple
rows with positions, or the elements of map `expr` into multiple rows and
columns with positions. Unless specified otherwise, uses the column name `pos`
for position, `col` for elements of the array or `key` and `value` for elements
of the map.",
@@ -519,34 +527,62 @@ object ExplodeOuterGeneratorBuilder extends
ExplodeGeneratorBuilderBase {
> SELECT _FUNC_(array(10,20));
0 10
1 20
- > SELECT * FROM _FUNC_(array(10,20));
+ > SELECT _FUNC_(collection => array(10,20));
0 10
1 20
""",
since = "2.0.0",
group = "generator_funcs")
// scalastyle:on line.size.limit line.contains.tab
-case class PosExplode(child: Expression) extends ExplodeBase {
- override val position = true
- override protected def withNewChildInternal(newChild: Expression):
PosExplode =
- copy(child = newChild)
+object PosExplodeExpressionBuilder extends ExpressionBuilder {
+ override def functionSignature: Option[FunctionSignature] =
+ Some(FunctionSignature(Seq(InputParameter("collection"))))
+
+ override def build(funcName: String, expressions: Seq[Expression]) :
Expression =
+ PosExplode(expressions(0))
}
-/**
- * Explodes an array of structs into a table.
- */
// scalastyle:off line.size.limit line.contains.tab
@ExpressionDescription(
- usage = "_FUNC_(expr) - Explodes an array of structs into a table. Uses
column names col1, col2, etc. by default unless specified otherwise.",
+ usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple
rows with positions, or the elements of map `expr` into multiple rows and
columns with positions. Unless specified otherwise, uses the column name `pos`
for position, `col` for elements of the array or `key` and `value` for elements
of the map.",
examples = """
Examples:
- > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b')));
- 1 a
- 2 b
+ > SELECT * FROM _FUNC_(array(10,20));
+ 0 10
+ 1 20
+ > SELECT * FROM _FUNC_(collection => array(10,20));
+ 0 10
+ 1 20
""",
- since = "2.0.0",
+ since = "3.5.0",
+ group = "generator_funcs")
+// scalastyle:on line.size.limit line.contains.tab
+object PosExplodeGeneratorBuilder extends PosExplodeGeneratorBuilderBase {
+ override def isOuter: Boolean = false
+}
+
+// scalastyle:off line.size.limit line.contains.tab
+@ExpressionDescription(
+ usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple
rows with positions, or the elements of map `expr` into multiple rows and
columns with positions. Unless specified otherwise, uses the column name `pos`
for position, `col` for elements of the array or `key` and `value` for elements
of the map.",
+ examples = """
+ Examples:
+ > SELECT * FROM _FUNC_(array(10,20));
+ 0 10
+ 1 20
+ > SELECT * FROM _FUNC_(collection => array(10,20));
+ 0 10
+ 1 20
+ """,
+ since = "3.5.0",
group = "generator_funcs")
// scalastyle:on line.size.limit line.contains.tab
+object PosExplodeOuterGeneratorBuilder extends PosExplodeGeneratorBuilderBase {
+ override def isOuter: Boolean = true
+}
+
+/**
+ * Explodes an array of structs into a table.
+ */
case class Inline(child: Expression) extends UnaryExpression with
CollectionGenerator {
override val inline: Boolean = true
override val position: Boolean = false
@@ -595,6 +631,76 @@ case class Inline(child: Expression) extends
UnaryExpression with CollectionGene
override protected def withNewChildInternal(newChild: Expression): Inline =
copy(child = newChild)
}
+trait InlineGeneratorBuilderBase extends GeneratorBuilder {
+ override def functionSignature: Option[FunctionSignature] =
+ Some(FunctionSignature(Seq(InputParameter("input"))))
+ override def buildGenerator(funcName: String, expressions: Seq[Expression]):
Generator = {
+ assert(expressions.size == 1)
+ Inline(expressions(0))
+ }
+}
+
+// scalastyle:off line.size.limit line.contains.tab
+@ExpressionDescription(
+ usage = "_FUNC_(expr) - Explodes an array of structs into a table. Uses
column names col1, col2, etc. by default unless specified otherwise.",
+ examples = """
+ Examples:
+ > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b')));
+ 1 a
+ 2 b
+ > SELECT _FUNC_(input => array(struct(1, 'a'), struct(2, 'b')));
+ 1 a
+ 2 b
+ """,
+ since = "2.0.0",
+ group = "generator_funcs")
+// scalastyle:on line.size.limit line.contains.tab
+object InlineExpressionBuilder extends ExpressionBuilder {
+ override def functionSignature: Option[FunctionSignature] =
+ Some(FunctionSignature(Seq(InputParameter("input"))))
+
+ override def build(funcName: String, expressions: Seq[Expression]) :
Expression =
+ Inline(expressions(0))
+}
+
+// scalastyle:off line.size.limit line.contains.tab
+@ExpressionDescription(
+ usage = "_FUNC_(expr) - Explodes an array of structs into a table. Uses
column names col1, col2, etc. by default unless specified otherwise.",
+ examples = """
+ Examples:
+ > SELECT * FROM _FUNC_(array(struct(1, 'a'), struct(2, 'b')));
+ 1 a
+ 2 b
+ > SELECT * FROM _FUNC_(input => array(struct(1, 'a'), struct(2, 'b')));
+ 1 a
+ 2 b
+ """,
+ since = "3.4.0",
+ group = "generator_funcs")
+// scalastyle:on line.size.limit line.contains.tab
+object InlineGeneratorBuilder extends InlineGeneratorBuilderBase {
+ override def isOuter: Boolean = false
+}
+
+// scalastyle:off line.size.limit line.contains.tab
+@ExpressionDescription(
+ usage = "_FUNC_(expr) - Explodes an array of structs into a table. Uses
column names col1, col2, etc. by default unless specified otherwise.",
+ examples = """
+ Examples:
+ > SELECT * FROM _FUNC_(array(struct(1, 'a'), struct(2, 'b')));
+ 1 a
+ 2 b
+ > SELECT * FROM _FUNC_(input => array(struct(1, 'a'), struct(2, 'b')));
+ 1 a
+ 2 b
+ """,
+ since = "3.4.0",
+ group = "generator_funcs")
+// scalastyle:on line.size.limit line.contains.tab
+object InlineOuterGeneratorBuilder extends InlineGeneratorBuilderBase {
+ override def isOuter: Boolean = true
+}
+
@ExpressionDescription(
usage = """_FUNC_() - Get Spark SQL keywords""",
examples = """
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
index d9b001fd7a84..67cdc0aa7a95 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
@@ -23,8 +23,7 @@ import scala.util.parsing.combinator.RegexParsers
import org.apache.spark.SparkRuntimeException
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.ExpressionBuilder
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.{ExpressionBuilder,
GeneratorBuilder, TypeCheckResult}
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
import org.apache.spark.sql.catalyst.expressions._
import
org.apache.spark.sql.catalyst.expressions.aggregate.{ImperativeAggregate,
TypedImperativeAggregate}
@@ -32,6 +31,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke
import org.apache.spark.sql.catalyst.json.JsonInferSchema
+import org.apache.spark.sql.catalyst.plans.logical.{FunctionSignature,
InputParameter}
import org.apache.spark.sql.catalyst.trees.TreePattern.{TreePattern,
VARIANT_GET}
import org.apache.spark.sql.catalyst.trees.UnaryLike
import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData,
GenericArrayData, QuotingUtils}
@@ -611,21 +611,6 @@ object VariantGetExpressionBuilder extends
VariantGetExpressionBuilderBase(true)
// scalastyle:on line.size.limit
object TryVariantGetExpressionBuilder extends
VariantGetExpressionBuilderBase(false)
-// scalastyle:off line.size.limit line.contains.tab
-@ExpressionDescription(
- usage = "_FUNC_(expr) - It separates a variant object/array into multiple
rows containing its fields/elements. Its result schema is `struct<pos int, key
string, value variant>`. `pos` is the position of the field/element in its
parent object/array, and `value` is the field/element value. `key` is the field
name when exploding a variant object, or is NULL when exploding a variant
array. It ignores any input that is not a variant array/object, including SQL
NULL, variant null, and any ot [...]
- examples = """
- Examples:
- > SELECT * from _FUNC_(parse_json('["hello", "world"]'));
- 0 NULL "hello"
- 1 NULL "world"
- > SELECT * from _FUNC_(parse_json('{"a": true, "b": 3.14}'));
- 0 a true
- 1 b 3.14
- """,
- since = "4.0.0",
- group = "variant_funcs")
-// scalastyle:on line.size.limit line.contains.tab
case class VariantExplode(child: Expression) extends UnaryExpression with
Generator
with ExpectsInputTypes {
override def inputTypes: Seq[AbstractDataType] = Seq(VariantType)
@@ -659,6 +644,53 @@ case class VariantExplode(child: Expression) extends
UnaryExpression with Genera
}
}
+trait VariantExplodeGeneratorBuilderBase extends GeneratorBuilder {
+ override def functionSignature: Option[FunctionSignature] =
+ Some(FunctionSignature(Seq(InputParameter("input"))))
+ override def buildGenerator(funcName: String, expressions: Seq[Expression]):
Generator = {
+ assert(expressions.size == 1)
+ VariantExplode(expressions(0))
+ }
+}
+
+// scalastyle:off line.size.limit line.contains.tab
+@ExpressionDescription(
+ usage = "_FUNC_(expr) - It separates a variant object/array into multiple
rows containing its fields/elements. Its result schema is `struct<pos int, key
string, value variant>`. `pos` is the position of the field/element in its
parent object/array, and `value` is the field/element value. `key` is the field
name when exploding a variant object, or is NULL when exploding a variant
array. It ignores any input that is not a variant array/object, including SQL
NULL, variant null, and any ot [...]
+ examples = """
+ Examples:
+ > SELECT * from _FUNC_(parse_json('["hello", "world"]'));
+ 0 NULL "hello"
+ 1 NULL "world"
+ > SELECT * from _FUNC_(input => parse_json('{"a": true, "b": 3.14}'));
+ 0 a true
+ 1 b 3.14
+ """,
+ since = "4.0.0",
+ group = "variant_funcs")
+// scalastyle:on line.size.limit line.contains.tab
+object VariantExplodeGeneratorBuilder extends
VariantExplodeGeneratorBuilderBase {
+ override def isOuter: Boolean = false
+}
+
+// scalastyle:off line.size.limit line.contains.tab
+@ExpressionDescription(
+ usage = "_FUNC_(expr) - It separates a variant object/array into multiple
rows containing its fields/elements. Its result schema is `struct<pos int, key
string, value variant>`. `pos` is the position of the field/element in its
parent object/array, and `value` is the field/element value. `key` is the field
name when exploding a variant object, or is NULL when exploding a variant
array. It ignores any input that is not a variant array/object, including SQL
NULL, variant null, and any ot [...]
+ examples = """
+ Examples:
+ > SELECT * from _FUNC_(parse_json('["hello", "world"]'));
+ 0 NULL "hello"
+ 1 NULL "world"
+ > SELECT * from _FUNC_(input => parse_json('{"a": true, "b": 3.14}'));
+ 0 a true
+ 1 b 3.14
+ """,
+ since = "4.0.0",
+ group = "variant_funcs")
+// scalastyle:on line.size.limit line.contains.tab
+object VariantExplodeOuterGeneratorBuilder extends
VariantExplodeGeneratorBuilderBase {
+ override def isOuter: Boolean = true
+}
+
object VariantExplode {
/**
* The actual implementation of the `VariantExplode` expression. We check
`isNull` separately
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/named-function-arguments.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/named-function-arguments.sql.out
index a49d80912f08..2315a5f0678a 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/named-function-arguments.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/named-function-arguments.sql.out
@@ -149,6 +149,104 @@ Project [num#x, val#x, Spark AS Spark#x]
+- OneRowRelation
+-- !query
+SELECT * FROM posexplode(collection => array(1, 2))
+-- !query analysis
+Project [pos#x, col#x]
++- Generate posexplode(array(1, 2)), false, [pos#x, col#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT * FROM posexplode_outer(collection => map('a', 1, 'b', 2))
+-- !query analysis
+Project [pos#x, key#x, value#x]
++- Generate posexplode(map(a, 1, b, 2)), true, [pos#x, key#x, value#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT * FROM posexplode(array(1, 2)), posexplode(array(3, 4))
+-- !query analysis
+Project [pos#x, col#x, pos#x, col#x]
++- Join Inner
+ :- Generate posexplode(array(1, 2)), false, [pos#x, col#x]
+ : +- OneRowRelation
+ +- Generate posexplode(array(3, 4)), false, [pos#x, col#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT * FROM posexplode(array(1, 2)) AS t, LATERAL posexplode(array(3 *
t.col, 4 * t.col))
+-- !query analysis
+Project [pos#x, col#x, pos#x, col#x]
++- LateralJoin lateral-subquery#x [col#x && col#x], Inner
+ : +- Generate posexplode(array((3 * outer(col#x)), (4 * outer(col#x)))),
false, [pos#x, col#x]
+ : +- OneRowRelation
+ +- SubqueryAlias t
+ +- Generate posexplode(array(1, 2)), false, [pos#x, col#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT pos, num, val, 'Spark' FROM posexplode(map(1, 'a', 2, 'b')) AS t(pos,
num, val)
+-- !query analysis
+Project [pos#x, num#x, val#x, Spark AS Spark#x]
++- SubqueryAlias t
+ +- Project [pos#x AS pos#x, key#x AS num#x, value#x AS val#x]
+ +- Generate posexplode(map(1, a, 2, b)), false, [pos#x, key#x, value#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT * FROM inline(input => array(struct(1, 'a'), struct(2, 'b')))
+-- !query analysis
+Project [col1#x, col2#x]
++- Generate inline(array(struct(col1, 1, col2, a), struct(col1, 2, col2, b))),
false, [col1#x, col2#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT * FROM inline_outer(input => array(struct(1, 'a'), struct(2, 'b')))
+-- !query analysis
+Project [col1#x, col2#x]
++- Generate inline(array(struct(col1, 1, col2, a), struct(col1, 2, col2, b))),
true, [col1#x, col2#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT * FROM inline(array(struct(1, 'a'), struct(2, 'b'))),
inline(array(struct(3, 'c'), struct(4, 'd')))
+-- !query analysis
+Project [col1#x, col2#x, col1#x, col2#x]
++- Join Inner
+ :- Generate inline(array(struct(col1, 1, col2, a), struct(col1, 2, col2,
b))), false, [col1#x, col2#x]
+ : +- OneRowRelation
+ +- Generate inline(array(struct(col1, 3, col2, c), struct(col1, 4, col2,
d))), false, [col1#x, col2#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT * FROM inline(array(struct(1, 'a'), struct(2, 'b'))) AS t, LATERAL
inline(array(struct(3 * t.col1, 4 * t.col1)))
+-- !query analysis
+Project [col1#x, col2#x, col1#x, col2#x]
++- LateralJoin lateral-subquery#x [col1#x && col1#x], Inner
+ : +- Generate inline(array(struct(col1, (3 * outer(col1#x)), col2, (4 *
outer(col1#x))))), false, [col1#x, col2#x]
+ : +- OneRowRelation
+ +- SubqueryAlias t
+ +- Generate inline(array(struct(col1, 1, col2, a), struct(col1, 2, col2,
b))), false, [col1#x, col2#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT num, val, 'Spark' FROM inline(array(struct(1, 'a'), struct(2, 'b'))) AS
t(num, val)
+-- !query analysis
+Project [num#x, val#x, Spark AS Spark#x]
++- SubqueryAlias t
+ +- Project [col1#x AS num#x, col2#x AS val#x]
+ +- Generate inline(array(struct(col1, 1, col2, a), struct(col1, 2, col2,
b))), false, [col1#x, col2#x]
+ +- OneRowRelation
+
+
-- !query
SELECT * FROM explode(collection => explode(array(1)))
-- !query analysis
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/variant/named-function-arguments.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/variant/named-function-arguments.sql.out
new file mode 100644
index 000000000000..571ec68048cc
--- /dev/null
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/variant/named-function-arguments.sql.out
@@ -0,0 +1,48 @@
+-- Automatically generated by SQLQueryTestSuite
+-- !query
+SELECT * FROM variant_explode(input => parse_json('["hello", "world"]'))
+-- !query analysis
+Project [pos#x, key#x, value#x]
++- Generate variant_explode(parse_json(["hello", "world"], true)), false,
[pos#x, key#x, value#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT * FROM variant_explode_outer(input => parse_json('{"a": true, "b":
3.14}'))
+-- !query analysis
+Project [pos#x, key#x, value#x]
++- Generate variant_explode(parse_json({"a": true, "b": 3.14}, true)), true,
[pos#x, key#x, value#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT * FROM variant_explode(parse_json('["hello", "world"]')),
variant_explode(parse_json('{"a": true, "b": 3.14}'))
+-- !query analysis
+Project [pos#x, key#x, value#x, pos#x, key#x, value#x]
++- Join Inner
+ :- Generate variant_explode(parse_json(["hello", "world"], true)), false,
[pos#x, key#x, value#x]
+ : +- OneRowRelation
+ +- Generate variant_explode(parse_json({"a": true, "b": 3.14}, true)),
false, [pos#x, key#x, value#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT * FROM variant_explode(parse_json('{"a": ["hello", "world"], "b": {"x":
true, "y": 3.14}}')) AS t, LATERAL variant_explode(t.value)
+-- !query analysis
+Project [pos#x, key#x, value#x, pos#x, key#x, value#x]
++- LateralJoin lateral-subquery#x [value#x], Inner
+ : +- Generate variant_explode(outer(value#x)), false, [pos#x, key#x,
value#x]
+ : +- OneRowRelation
+ +- SubqueryAlias t
+ +- Generate variant_explode(parse_json({"a": ["hello", "world"], "b":
{"x": true, "y": 3.14}}, true)), false, [pos#x, key#x, value#x]
+ +- OneRowRelation
+
+
+-- !query
+SELECT num, key, val, 'Spark' FROM variant_explode(parse_json('["hello",
"world"]')) AS t(num, key, val)
+-- !query analysis
+Project [num#x, key#x, val#x, Spark AS Spark#x]
++- SubqueryAlias t
+ +- Project [pos#x AS num#x, key#x AS key#x, value#x AS val#x]
+ +- Generate variant_explode(parse_json(["hello", "world"], true)),
false, [pos#x, key#x, value#x]
+ +- OneRowRelation
diff --git
a/sql/core/src/test/resources/sql-tests/inputs/named-function-arguments.sql
b/sql/core/src/test/resources/sql-tests/inputs/named-function-arguments.sql
index 99f33d781525..a795a19828c1 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/named-function-arguments.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/named-function-arguments.sql
@@ -32,6 +32,20 @@ SELECT * FROM explode(array(1, 2)), explode(array(3, 4));
SELECT * FROM explode(array(1, 2)) AS t, LATERAL explode(array(3 * t.col, 4 *
t.col));
SELECT num, val, 'Spark' FROM explode(map(1, 'a', 2, 'b')) AS t(num, val);
+-- Test for tabled value functions posexplode and posexplode_outer
+SELECT * FROM posexplode(collection => array(1, 2));
+SELECT * FROM posexplode_outer(collection => map('a', 1, 'b', 2));
+SELECT * FROM posexplode(array(1, 2)), posexplode(array(3, 4));
+SELECT * FROM posexplode(array(1, 2)) AS t, LATERAL posexplode(array(3 *
t.col, 4 * t.col));
+SELECT pos, num, val, 'Spark' FROM posexplode(map(1, 'a', 2, 'b')) AS t(pos,
num, val);
+
+-- Test for tabled value functions inline and inline_outer
+SELECT * FROM inline(input => array(struct(1, 'a'), struct(2, 'b')));
+SELECT * FROM inline_outer(input => array(struct(1, 'a'), struct(2, 'b')));
+SELECT * FROM inline(array(struct(1, 'a'), struct(2, 'b'))),
inline(array(struct(3, 'c'), struct(4, 'd')));
+SELECT * FROM inline(array(struct(1, 'a'), struct(2, 'b'))) AS t, LATERAL
inline(array(struct(3 * t.col1, 4 * t.col1)));
+SELECT num, val, 'Spark' FROM inline(array(struct(1, 'a'), struct(2, 'b'))) AS
t(num, val);
+
-- Test for wrapped EXPLODE call to check error preservation
SELECT * FROM explode(collection => explode(array(1)));
SELECT * FROM explode(collection => explode(collection => array(1)));
diff --git
a/sql/core/src/test/resources/sql-tests/inputs/variant/named-function-arguments.sql
b/sql/core/src/test/resources/sql-tests/inputs/variant/named-function-arguments.sql
new file mode 100644
index 000000000000..e63e68169a81
--- /dev/null
+++
b/sql/core/src/test/resources/sql-tests/inputs/variant/named-function-arguments.sql
@@ -0,0 +1,6 @@
+-- Test for tabled value functions variant_explode and variant_explode_outer
+SELECT * FROM variant_explode(input => parse_json('["hello", "world"]'));
+SELECT * FROM variant_explode_outer(input => parse_json('{"a": true, "b":
3.14}'));
+SELECT * FROM variant_explode(parse_json('["hello", "world"]')),
variant_explode(parse_json('{"a": true, "b": 3.14}'));
+SELECT * FROM variant_explode(parse_json('{"a": ["hello", "world"], "b": {"x":
true, "y": 3.14}}')) AS t, LATERAL variant_explode(t.value);
+SELECT num, key, val, 'Spark' FROM variant_explode(parse_json('["hello",
"world"]')) AS t(num, key, val);
diff --git
a/sql/core/src/test/resources/sql-tests/results/named-function-arguments.sql.out
b/sql/core/src/test/resources/sql-tests/results/named-function-arguments.sql.out
index 6dbc20b7153e..e5063dc0cf31 100644
---
a/sql/core/src/test/resources/sql-tests/results/named-function-arguments.sql.out
+++
b/sql/core/src/test/resources/sql-tests/results/named-function-arguments.sql.out
@@ -126,6 +126,102 @@ struct<num:int,val:string,Spark:string>
2 b Spark
+-- !query
+SELECT * FROM posexplode(collection => array(1, 2))
+-- !query schema
+struct<pos:int,col:int>
+-- !query output
+0 1
+1 2
+
+
+-- !query
+SELECT * FROM posexplode_outer(collection => map('a', 1, 'b', 2))
+-- !query schema
+struct<pos:int,key:string,value:int>
+-- !query output
+0 a 1
+1 b 2
+
+
+-- !query
+SELECT * FROM posexplode(array(1, 2)), posexplode(array(3, 4))
+-- !query schema
+struct<pos:int,col:int,pos:int,col:int>
+-- !query output
+0 1 0 3
+0 1 1 4
+1 2 0 3
+1 2 1 4
+
+
+-- !query
+SELECT * FROM posexplode(array(1, 2)) AS t, LATERAL posexplode(array(3 *
t.col, 4 * t.col))
+-- !query schema
+struct<pos:int,col:int,pos:int,col:int>
+-- !query output
+0 1 0 3
+0 1 1 4
+1 2 0 6
+1 2 1 8
+
+
+-- !query
+SELECT pos, num, val, 'Spark' FROM posexplode(map(1, 'a', 2, 'b')) AS t(pos,
num, val)
+-- !query schema
+struct<pos:int,num:int,val:string,Spark:string>
+-- !query output
+0 1 a Spark
+1 2 b Spark
+
+
+-- !query
+SELECT * FROM inline(input => array(struct(1, 'a'), struct(2, 'b')))
+-- !query schema
+struct<col1:int,col2:string>
+-- !query output
+1 a
+2 b
+
+
+-- !query
+SELECT * FROM inline_outer(input => array(struct(1, 'a'), struct(2, 'b')))
+-- !query schema
+struct<col1:int,col2:string>
+-- !query output
+1 a
+2 b
+
+
+-- !query
+SELECT * FROM inline(array(struct(1, 'a'), struct(2, 'b'))),
inline(array(struct(3, 'c'), struct(4, 'd')))
+-- !query schema
+struct<col1:int,col2:string,col1:int,col2:string>
+-- !query output
+1 a 3 c
+1 a 4 d
+2 b 3 c
+2 b 4 d
+
+
+-- !query
+SELECT * FROM inline(array(struct(1, 'a'), struct(2, 'b'))) AS t, LATERAL
inline(array(struct(3 * t.col1, 4 * t.col1)))
+-- !query schema
+struct<col1:int,col2:string,col1:int,col2:int>
+-- !query output
+1 a 3 4
+2 b 6 8
+
+
+-- !query
+SELECT num, val, 'Spark' FROM inline(array(struct(1, 'a'), struct(2, 'b'))) AS
t(num, val)
+-- !query schema
+struct<num:int,val:string,Spark:string>
+-- !query output
+1 a Spark
+2 b Spark
+
+
-- !query
SELECT * FROM explode(collection => explode(array(1)))
-- !query schema
diff --git
a/sql/core/src/test/resources/sql-tests/results/variant/named-function-arguments.sql.out
b/sql/core/src/test/resources/sql-tests/results/variant/named-function-arguments.sql.out
new file mode 100644
index 000000000000..10a439831fbd
--- /dev/null
+++
b/sql/core/src/test/resources/sql-tests/results/variant/named-function-arguments.sql.out
@@ -0,0 +1,48 @@
+-- Automatically generated by SQLQueryTestSuite
+-- !query
+SELECT * FROM variant_explode(input => parse_json('["hello", "world"]'))
+-- !query schema
+struct<pos:int,key:string,value:variant>
+-- !query output
+0 NULL "hello"
+1 NULL "world"
+
+
+-- !query
+SELECT * FROM variant_explode_outer(input => parse_json('{"a": true, "b":
3.14}'))
+-- !query schema
+struct<pos:int,key:string,value:variant>
+-- !query output
+0 a true
+1 b 3.14
+
+
+-- !query
+SELECT * FROM variant_explode(parse_json('["hello", "world"]')),
variant_explode(parse_json('{"a": true, "b": 3.14}'))
+-- !query schema
+struct<pos:int,key:string,value:variant,pos:int,key:string,value:variant>
+-- !query output
+0 NULL "hello" 0 a true
+0 NULL "hello" 1 b 3.14
+1 NULL "world" 0 a true
+1 NULL "world" 1 b 3.14
+
+
+-- !query
+SELECT * FROM variant_explode(parse_json('{"a": ["hello", "world"], "b": {"x":
true, "y": 3.14}}')) AS t, LATERAL variant_explode(t.value)
+-- !query schema
+struct<pos:int,key:string,value:variant,pos:int,key:string,value:variant>
+-- !query output
+0 a ["hello","world"] 0 NULL "hello"
+0 a ["hello","world"] 1 NULL "world"
+1 b {"x":true,"y":3.14} 0 x true
+1 b {"x":true,"y":3.14} 1 y 3.14
+
+
+-- !query
+SELECT num, key, val, 'Spark' FROM variant_explode(parse_json('["hello",
"world"]')) AS t(num, key, val)
+-- !query schema
+struct<num:int,key:string,val:variant,Spark:string>
+-- !query output
+0 NULL "hello" Spark
+1 NULL "world" Spark
diff --git
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index 331572e62f56..782f549182ec 100644
---
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -104,7 +104,9 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite
with SharedThriftServ
"timestampNTZ/datetime-special-ansi.sql",
// SPARK-47264
"collations.sql",
- "pipe-operators.sql"
+ "pipe-operators.sql",
+ // VARIANT type
+ "variant/named-function-arguments.sql"
)
override def runQueries(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
