gene-db commented on code in PR #57827:
URL: https://github.com/apache/spark/pull/57827#discussion_r3896306801
##########
sql/api/src/main/scala/org/apache/spark/sql/TableValuedFunction.scala:
##########
@@ -169,6 +169,22 @@ abstract class TableValuedFunction {
*/
def variant_explode(input: Column): Dataset[Row]
+ /**
+ * Separates a variant object/array into multiple rows, including nested
fields/elements when
+ * `recursive` is true. When false, the result schema is
+ * `struct<pos int, key string, value variant>`; when true, the result
schema is
+ * `struct<path string, pos int, key string, value variant>`.
+ *
+ * `pos` is the position within the parent, `key` is the object field name
or NULL for arrays,
+ * and `value` is the field/element value. In recursive mode, `path` is its
JSONPath. It ignores
Review Comment:
Is the path in the canonicalized json path format?
##########
sql/core/src/test/scala/org/apache/spark/sql/DataFrameTableValuedFunctionsSuite.scala:
##########
@@ -528,6 +528,21 @@ class DataFrameTableValuedFunctionsSuite extends
SharedSparkSession {
}
}
+ test("variant_explode - recursive") {
+ val nested = parse_json(lit("""{"a": [1, {"b": 2}]}"""))
+ val actual = spark.tvf.variant_explode(nested, recursive = true)
Review Comment:
Should we try the 2 arg function, but with `recursive = false`?
##########
sql/api/src/main/scala/org/apache/spark/sql/TableValuedFunction.scala:
##########
@@ -169,6 +169,22 @@ abstract class TableValuedFunction {
*/
def variant_explode(input: Column): Dataset[Row]
+ /**
+ * Separates a variant object/array into multiple rows, including nested
fields/elements when
+ * `recursive` is true. When false, the result schema is
+ * `struct<pos int, key string, value variant>`; when true, the result
schema is
+ * `struct<path string, pos int, key string, value variant>`.
+ *
+ * `pos` is the position within the parent, `key` is the object field name
or NULL for arrays,
+ * and `value` is the field/element value. In recursive mode, `path` is its
JSONPath. It ignores
+ * any input that is not a variant array/object, including SQL NULL, variant
null, and any other
+ * variant values.
+ *
+ * @group variant_funcs
+ * @since 4.3.0
Review Comment:
Is this the right version to specify?
##########
sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala:
##########
@@ -1056,6 +1056,84 @@ class VariantSuite extends SharedSparkSession with
ExpressionEvalHelper {
check("""{"a": [1, 2, 3], "b": true}""", Seq(Row(0, "a", "[1,2,3]"),
Row(1, "b", "true")))
check("""[null, "hello", {}]""",
Seq(Row(0, null, "null"), Row(1, null, "\"hello\""), Row(2, null,
"{}")))
+
+ val nonRecursive = sql(
+ """SELECT * FROM variant_explode(parse_json('{"a": [1]}'), false)""")
+ assert(nonRecursive.schema.fieldNames.toSeq == Seq("pos", "key",
"value"))
+ checkAnswer(
+ nonRecursive.selectExpr("pos", "key", "to_json(value)"),
+ Seq(Row(0, "a", "[1]")))
+
+ val recursiveExpected = Seq(
+ Row("$.a", 0, "a", """[1,{"b":2}]"""),
+ Row("$.a[0]", 0, null, "1"),
+ Row("$.a[1]", 1, null, """{"b":2}"""),
+ Row("$.a[1].b", 0, "b", "2"),
+ Row("$.c", 1, "c", """{"d":[3]}"""),
+ Row("$.c.d", 0, "d", "[3]"),
+ Row("$.c.d[0]", 0, null, "3"))
+ Seq("variant_explode", "variant_explode_outer").foreach { function =>
Review Comment:
Can we add tests where the paths have special characters?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
